From 107137a4365c416587c3fbc1e58be07c8bdd6a26 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:51:28 -0700 Subject: [PATCH 01/36] chore: re-vendor kiln_server SDK from dchiang/synthetic-user@1d57386 Removes the /respond SDK module and its supporting wire types (RespondRequest/Response, SyntheticUserDriverConfig, ConversationTurn, the nested SyntheticUserInfo model). Per-turn synthetic-user invocation moves to OSS at libs/core/kiln_ai/synthetic_user/ in a subsequent commit. Collapses SyntheticUserCase.synthetic_user_info to a single tagged blob string: ......... The server treats the blob as opaque; the local player parses it. Adds a typed `code` literal on /generate's 502 response (llm_unavailable | upstream_invalid_output) so callers can discriminate between transient model failures and unparseable model output. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...jobs_prompt_optimization_job_start_post.py | 2 + ...ample_job_v1_jobs_sample_job_start_post.py | 1 - .../api/synthetic_user/__init__.py | 1 + ...enerate_v1_synthetic_user_generate_post.py | 253 ++++++++++++++++++ .../kiln_ai_server_client/models/__init__.py | 18 ++ ...pletion_assistant_message_param_wrapper.py | 53 ++++ .../models/chat_session_list_item.py | 3 +- .../generate_synthetic_users_request.py | 85 ++++++ .../generate_synthetic_users_response.py | 82 ++++++ ...nthetic_user_generate_post_response_401.py | 72 +++++ ...nthetic_user_generate_post_response_500.py | 72 +++++ ...nthetic_user_generate_post_response_502.py | 73 +++++ ...ic_user_generate_post_response_502_code.py | 9 + .../models/kiln_base_model.py | 3 +- .../models/message_usage.py | 164 ++++++++++++ .../models/synthetic_user_case.py | 72 +++++ .../models/task_output.py | 3 +- .../models/task_output_rating.py | 3 +- .../kiln_ai_server_client/models/task_run.py | 38 ++- .../kiln_ai_server_client/models/usage.py | 44 ++- 20 files changed, 1031 insertions(+), 20 deletions(-) create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/__init__.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/generate_v1_synthetic_user_generate_post.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_request.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_401.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_500.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502_code.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/message_usage.py create mode 100644 app/desktop/studio_server/api_client/kiln_ai_server_client/models/synthetic_user_case.py diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py index 8b282ed871..ba1ec9502d 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py @@ -26,6 +26,8 @@ def _get_kwargs( _kwargs["files"] = body.to_multipart() + headers["Content-Type"] = "multipart/form-data; boundary=+++" + _kwargs["headers"] = headers return _kwargs diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_sample_job_v1_jobs_sample_job_start_post.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_sample_job_v1_jobs_sample_job_start_post.py index 0cf5080c05..69abed29c0 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_sample_job_v1_jobs_sample_job_start_post.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_sample_job_v1_jobs_sample_job_start_post.py @@ -23,7 +23,6 @@ def _get_kwargs( } _kwargs["data"] = body.to_dict() - headers["Content-Type"] = "application/x-www-form-urlencoded" _kwargs["headers"] = headers diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/__init__.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/__init__.py new file mode 100644 index 0000000000..2d7c0b23da --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/__init__.py @@ -0,0 +1 @@ +"""Contains endpoint functions for accessing the API""" diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/generate_v1_synthetic_user_generate_post.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/generate_v1_synthetic_user_generate_post.py new file mode 100644 index 0000000000..23d2dd4051 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/generate_v1_synthetic_user_generate_post.py @@ -0,0 +1,253 @@ +from http import HTTPStatus +from typing import Any + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.generate_synthetic_users_request import GenerateSyntheticUsersRequest +from ...models.generate_synthetic_users_response import GenerateSyntheticUsersResponse +from ...models.generate_v1_synthetic_user_generate_post_response_401 import ( + GenerateV1SyntheticUserGeneratePostResponse401, +) +from ...models.generate_v1_synthetic_user_generate_post_response_500 import ( + GenerateV1SyntheticUserGeneratePostResponse500, +) +from ...models.generate_v1_synthetic_user_generate_post_response_502 import ( + GenerateV1SyntheticUserGeneratePostResponse502, +) +from ...models.http_validation_error import HTTPValidationError +from ...types import Response + + +def _get_kwargs( + *, + body: GenerateSyntheticUsersRequest, +) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/v1/synthetic_user/generate", + } + + _kwargs["json"] = body.to_dict() + + headers["Content-Type"] = "application/json" + + _kwargs["headers"] = headers + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> ( + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError + | None +): + if response.status_code == 200: + response_200 = GenerateSyntheticUsersResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 401: + response_401 = GenerateV1SyntheticUserGeneratePostResponse401.from_dict(response.json()) + + return response_401 + + if response.status_code == 422: + response_422 = HTTPValidationError.from_dict(response.json()) + + return response_422 + + if response.status_code == 500: + response_500 = GenerateV1SyntheticUserGeneratePostResponse500.from_dict(response.json()) + + return response_500 + + if response.status_code == 502: + response_502 = GenerateV1SyntheticUserGeneratePostResponse502.from_dict(response.json()) + + return response_502 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[ + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError +]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + *, + client: AuthenticatedClient, + body: GenerateSyntheticUsersRequest, +) -> Response[ + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError +]: + """Generate + + Return `num_cases` synthetic-user cases for the authoring UX. + + Args: + body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate. + + Generates `num_cases` synthetic-user cases designed to probe + `target_specification` against the agent described by `target_task_prompt`, + across multi-turn conversations. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError] + """ + + kwargs = _get_kwargs( + body=body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + *, + client: AuthenticatedClient, + body: GenerateSyntheticUsersRequest, +) -> ( + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError + | None +): + """Generate + + Return `num_cases` synthetic-user cases for the authoring UX. + + Args: + body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate. + + Generates `num_cases` synthetic-user cases designed to probe + `target_specification` against the agent described by `target_task_prompt`, + across multi-turn conversations. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError + """ + + return sync_detailed( + client=client, + body=body, + ).parsed + + +async def asyncio_detailed( + *, + client: AuthenticatedClient, + body: GenerateSyntheticUsersRequest, +) -> Response[ + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError +]: + """Generate + + Return `num_cases` synthetic-user cases for the authoring UX. + + Args: + body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate. + + Generates `num_cases` synthetic-user cases designed to probe + `target_specification` against the agent described by `target_task_prompt`, + across multi-turn conversations. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError] + """ + + kwargs = _get_kwargs( + body=body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + *, + client: AuthenticatedClient, + body: GenerateSyntheticUsersRequest, +) -> ( + GenerateSyntheticUsersResponse + | GenerateV1SyntheticUserGeneratePostResponse401 + | GenerateV1SyntheticUserGeneratePostResponse500 + | GenerateV1SyntheticUserGeneratePostResponse502 + | HTTPValidationError + | None +): + """Generate + + Return `num_cases` synthetic-user cases for the authoring UX. + + Args: + body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate. + + Generates `num_cases` synthetic-user cases designed to probe + `target_specification` against the agent described by `target_task_prompt`, + across multi-turn conversations. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError + """ + + return ( + await asyncio_detailed( + client=client, + body=body, + ) + ).parsed diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/__init__.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/__init__.py index 2fc0ec05a9..b5814fc943 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/__init__.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/__init__.py @@ -55,6 +55,14 @@ from .generate_batch_input import GenerateBatchInput from .generate_batch_output import GenerateBatchOutput from .generate_batch_output_data_by_topic import GenerateBatchOutputDataByTopic +from .generate_synthetic_users_request import GenerateSyntheticUsersRequest +from .generate_synthetic_users_response import GenerateSyntheticUsersResponse +from .generate_v1_synthetic_user_generate_post_response_401 import GenerateV1SyntheticUserGeneratePostResponse401 +from .generate_v1_synthetic_user_generate_post_response_500 import GenerateV1SyntheticUserGeneratePostResponse500 +from .generate_v1_synthetic_user_generate_post_response_502 import GenerateV1SyntheticUserGeneratePostResponse502 +from .generate_v1_synthetic_user_generate_post_response_502_code import ( + GenerateV1SyntheticUserGeneratePostResponse502Code, +) from .get_session_v1_chat_sessions_session_id_get_response_400 import GetSessionV1ChatSessionsSessionIdGetResponse400 from .get_session_v1_chat_sessions_session_id_get_response_404 import GetSessionV1ChatSessionsSessionIdGetResponse404 from .get_session_v1_chat_sessions_session_id_get_response_426 import GetSessionV1ChatSessionsSessionIdGetResponse426 @@ -83,6 +91,7 @@ from .mcp_tool_reference import MCPToolReference from .mcp_tool_reference_input_schema_type_0 import MCPToolReferenceInputSchemaType0 from .mcp_tool_reference_output_schema_type_0 import MCPToolReferenceOutputSchemaType0 +from .message_usage import MessageUsage from .model_provider_name import ModelProviderName from .new_proposed_spec_edit_api import NewProposedSpecEditApi from .output_file_info import OutputFileInfo @@ -110,6 +119,7 @@ from .synthetic_data_generation_session_config_input import SyntheticDataGenerationSessionConfigInput from .synthetic_data_generation_step_config import SyntheticDataGenerationStepConfig from .synthetic_data_generation_step_config_input import SyntheticDataGenerationStepConfigInput +from .synthetic_user_case import SyntheticUserCase from .task_info import TaskInfo from .task_metadata import TaskMetadata from .task_output import TaskOutput @@ -166,6 +176,12 @@ "GenerateBatchInput", "GenerateBatchOutput", "GenerateBatchOutputDataByTopic", + "GenerateSyntheticUsersRequest", + "GenerateSyntheticUsersResponse", + "GenerateV1SyntheticUserGeneratePostResponse401", + "GenerateV1SyntheticUserGeneratePostResponse500", + "GenerateV1SyntheticUserGeneratePostResponse502", + "GenerateV1SyntheticUserGeneratePostResponse502Code", "GetSessionV1ChatSessionsSessionIdGetResponse400", "GetSessionV1ChatSessionsSessionIdGetResponse404", "GetSessionV1ChatSessionsSessionIdGetResponse426", @@ -194,6 +210,7 @@ "MCPToolReference", "MCPToolReferenceInputSchemaType0", "MCPToolReferenceOutputSchemaType0", + "MessageUsage", "ModelProviderName", "NewProposedSpecEditApi", "OutputFileInfo", @@ -221,6 +238,7 @@ "SyntheticDataGenerationSessionConfigInput", "SyntheticDataGenerationStepConfig", "SyntheticDataGenerationStepConfigInput", + "SyntheticUserCase", "TaskInfo", "TaskMetadata", "TaskOutput", diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_completion_assistant_message_param_wrapper.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_completion_assistant_message_param_wrapper.py index dba1f05d97..f9b0a4f0fe 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_completion_assistant_message_param_wrapper.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_completion_assistant_message_param_wrapper.py @@ -14,6 +14,7 @@ from ..models.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam from ..models.chat_completion_message_function_tool_call_param import ChatCompletionMessageFunctionToolCallParam from ..models.function_call import FunctionCall + from ..models.message_usage import MessageUsage T = TypeVar("T", bound="ChatCompletionAssistantMessageParamWrapper") @@ -37,6 +38,8 @@ class ChatCompletionAssistantMessageParamWrapper: name (str | Unset): refusal (None | str | Unset): tool_calls (list[ChatCompletionMessageFunctionToolCallParam] | Unset): + latency_ms (int | None | Unset): + usage (MessageUsage | None | Unset): """ role: Literal["assistant"] @@ -49,12 +52,15 @@ class ChatCompletionAssistantMessageParamWrapper: name: str | Unset = UNSET refusal: None | str | Unset = UNSET tool_calls: list[ChatCompletionMessageFunctionToolCallParam] | Unset = UNSET + latency_ms: int | None | Unset = UNSET + usage: MessageUsage | None | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: from ..models.audio import Audio from ..models.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam from ..models.function_call import FunctionCall + from ..models.message_usage import MessageUsage role = self.role @@ -112,6 +118,20 @@ def to_dict(self) -> dict[str, Any]: tool_calls_item = tool_calls_item_data.to_dict() tool_calls.append(tool_calls_item) + latency_ms: int | None | Unset + if isinstance(self.latency_ms, Unset): + latency_ms = UNSET + else: + latency_ms = self.latency_ms + + usage: dict[str, Any] | None | Unset + if isinstance(self.usage, Unset): + usage = UNSET + elif isinstance(self.usage, MessageUsage): + usage = self.usage.to_dict() + else: + usage = self.usage + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -133,6 +153,10 @@ def to_dict(self) -> dict[str, Any]: field_dict["refusal"] = refusal if tool_calls is not UNSET: field_dict["tool_calls"] = tool_calls + if latency_ms is not UNSET: + field_dict["latency_ms"] = latency_ms + if usage is not UNSET: + field_dict["usage"] = usage return field_dict @@ -143,6 +167,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam from ..models.chat_completion_message_function_tool_call_param import ChatCompletionMessageFunctionToolCallParam from ..models.function_call import FunctionCall + from ..models.message_usage import MessageUsage d = dict(src_dict) role = cast(Literal["assistant"], d.pop("role")) @@ -257,6 +282,32 @@ def _parse_refusal(data: object) -> None | str | Unset: tool_calls.append(tool_calls_item) + def _parse_latency_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + latency_ms = _parse_latency_ms(d.pop("latency_ms", UNSET)) + + def _parse_usage(data: object) -> MessageUsage | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + usage_type_0 = MessageUsage.from_dict(data) + + return usage_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(MessageUsage | None | Unset, data) + + usage = _parse_usage(d.pop("usage", UNSET)) + chat_completion_assistant_message_param_wrapper = cls( role=role, audio=audio, @@ -266,6 +317,8 @@ def _parse_refusal(data: object) -> None | str | Unset: name=name, refusal=refusal, tool_calls=tool_calls, + latency_ms=latency_ms, + usage=usage, ) chat_completion_assistant_message_param_wrapper.additional_properties = d diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_session_list_item.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_session_list_item.py index 7a93d1a7d2..b0ae1f24f1 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_session_list_item.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/chat_session_list_item.py @@ -6,7 +6,6 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -from dateutil.parser import isoparse T = TypeVar("T", bound="ChatSessionListItem") @@ -54,7 +53,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: d = dict(src_dict) id = d.pop("id") - updated_at = isoparse(d.pop("updated_at")) + updated_at = datetime.datetime.fromisoformat(d.pop("updated_at")) title = d.pop("title") diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_request.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_request.py new file mode 100644 index 0000000000..537e01d1a4 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_request.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="GenerateSyntheticUsersRequest") + + +@_attrs_define +class GenerateSyntheticUsersRequest: + """Request body for POST /v1/synthetic_user/generate. + + Generates `num_cases` synthetic-user cases designed to probe + `target_specification` against the agent described by `target_task_prompt`, + across multi-turn conversations. + + Attributes: + target_task_prompt (str): Complete prompt of the target task (the AI assistant under evaluation). Used as + material for designing realistic synthetic users; never executed by this endpoint. + target_specification (str): Behavior or issue to investigate about the target task (e.g. 'hallucinates tax-year- + specific rules for years before 2018'). Generated cases are designed so a multi-turn conversation will naturally + surface this behavior. + num_cases (int): Number of synthetic-user cases to generate (1-50). + """ + + target_task_prompt: str + target_specification: str + num_cases: int + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + target_task_prompt = self.target_task_prompt + + target_specification = self.target_specification + + num_cases = self.num_cases + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "target_task_prompt": target_task_prompt, + "target_specification": target_specification, + "num_cases": num_cases, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + target_task_prompt = d.pop("target_task_prompt") + + target_specification = d.pop("target_specification") + + num_cases = d.pop("num_cases") + + generate_synthetic_users_request = cls( + target_task_prompt=target_task_prompt, + target_specification=target_specification, + num_cases=num_cases, + ) + + generate_synthetic_users_request.additional_properties = d + return generate_synthetic_users_request + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py new file mode 100644 index 0000000000..4ef08c2b99 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.synthetic_user_case import SyntheticUserCase + + +T = TypeVar("T", bound="GenerateSyntheticUsersResponse") + + +@_attrs_define +class GenerateSyntheticUsersResponse: + """Response body for POST /v1/synthetic_user/generate. + + `cases` is the strict-N batch contract: exactly `num_cases` cases on + success. If any single generated case is malformed (e.g. an empty + `synthetic_user_info` blob), the whole batch fails with HTTP 502 + `upstream_invalid_output` — the server does not return a partial batch. + Clients should re-call to retry. + + Attributes: + cases (list[SyntheticUserCase]): + """ + + cases: list[SyntheticUserCase] + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + cases = [] + for cases_item_data in self.cases: + cases_item = cases_item_data.to_dict() + cases.append(cases_item) + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "cases": cases, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.synthetic_user_case import SyntheticUserCase + + d = dict(src_dict) + cases = [] + _cases = d.pop("cases") + for cases_item_data in _cases: + cases_item = SyntheticUserCase.from_dict(cases_item_data) + + cases.append(cases_item) + + generate_synthetic_users_response = cls( + cases=cases, + ) + + generate_synthetic_users_response.additional_properties = d + return generate_synthetic_users_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_401.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_401.py new file mode 100644 index 0000000000..449862a4e2 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_401.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GenerateV1SyntheticUserGeneratePostResponse401") + + +@_attrs_define +class GenerateV1SyntheticUserGeneratePostResponse401: + """ + Attributes: + message (str): + code (str | Unset): + """ + + message: str + code: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message + + code = self.code + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + } + ) + if code is not UNSET: + field_dict["code"] = code + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + message = d.pop("message") + + code = d.pop("code", UNSET) + + generate_v1_synthetic_user_generate_post_response_401 = cls( + message=message, + code=code, + ) + + generate_v1_synthetic_user_generate_post_response_401.additional_properties = d + return generate_v1_synthetic_user_generate_post_response_401 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_500.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_500.py new file mode 100644 index 0000000000..3dfa393ba9 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_500.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GenerateV1SyntheticUserGeneratePostResponse500") + + +@_attrs_define +class GenerateV1SyntheticUserGeneratePostResponse500: + """ + Attributes: + message (str): + code (str | Unset): + """ + + message: str + code: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message + + code = self.code + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + } + ) + if code is not UNSET: + field_dict["code"] = code + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + message = d.pop("message") + + code = d.pop("code", UNSET) + + generate_v1_synthetic_user_generate_post_response_500 = cls( + message=message, + code=code, + ) + + generate_v1_synthetic_user_generate_post_response_500.additional_properties = d + return generate_v1_synthetic_user_generate_post_response_500 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502.py new file mode 100644 index 0000000000..fa17fda3a6 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.generate_v1_synthetic_user_generate_post_response_502_code import ( + GenerateV1SyntheticUserGeneratePostResponse502Code, +) + +T = TypeVar("T", bound="GenerateV1SyntheticUserGeneratePostResponse502") + + +@_attrs_define +class GenerateV1SyntheticUserGeneratePostResponse502: + """ + Attributes: + message (str): + code (GenerateV1SyntheticUserGeneratePostResponse502Code): + """ + + message: str + code: GenerateV1SyntheticUserGeneratePostResponse502Code + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message + + code = self.code.value + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + "code": code, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + message = d.pop("message") + + code = GenerateV1SyntheticUserGeneratePostResponse502Code(d.pop("code")) + + generate_v1_synthetic_user_generate_post_response_502 = cls( + message=message, + code=code, + ) + + generate_v1_synthetic_user_generate_post_response_502.additional_properties = d + return generate_v1_synthetic_user_generate_post_response_502 + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502_code.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502_code.py new file mode 100644 index 0000000000..d91cbe1e6b --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_v1_synthetic_user_generate_post_response_502_code.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class GenerateV1SyntheticUserGeneratePostResponse502Code(str, Enum): + LLM_UNAVAILABLE = "llm_unavailable" + UPSTREAM_INVALID_OUTPUT = "upstream_invalid_output" + + def __str__(self) -> str: + return str(self.value) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/kiln_base_model.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/kiln_base_model.py index 7a03b636eb..74084a998a 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/kiln_base_model.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/kiln_base_model.py @@ -6,7 +6,6 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -from dateutil.parser import isoparse from ..types import UNSET, Unset @@ -115,7 +114,7 @@ def _parse_path(data: object) -> None | str | Unset: if isinstance(_created_at, Unset): created_at = UNSET else: - created_at = isoparse(_created_at) + created_at = datetime.datetime.fromisoformat(_created_at) created_by = d.pop("created_by", UNSET) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/message_usage.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/message_usage.py new file mode 100644 index 0000000000..65157b11b8 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/message_usage.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="MessageUsage") + + +@_attrs_define +class MessageUsage: + """Token usage and cost for a single LLM call or a multi-message sum. + + Carries only the fields that are meaningfully aggregatable across + messages: token counts and cost. Per-call latency lives on the + individual message's ``latency_ms`` field; aggregating it across the + full trace would mix latencies from different points in time, so + ``MessageUsage`` does NOT carry ``total_llm_latency_ms``. + + The :class:`Usage` subclass adds ``total_llm_latency_ms`` for the + in-flight per-run accumulator that tracks how long this run spent + waiting on LLM calls. + + Attributes: + input_tokens (int | None | Unset): The number of input tokens used. + output_tokens (int | None | Unset): The number of output tokens used. + total_tokens (int | None | Unset): The total number of tokens used. + cost (float | None | Unset): The cost in US dollars, saved at runtime (prices can change over time). + cached_tokens (int | None | Unset): Number of tokens served from prompt cache. None if not reported. + """ + + input_tokens: int | None | Unset = UNSET + output_tokens: int | None | Unset = UNSET + total_tokens: int | None | Unset = UNSET + cost: float | None | Unset = UNSET + cached_tokens: int | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + input_tokens: int | None | Unset + if isinstance(self.input_tokens, Unset): + input_tokens = UNSET + else: + input_tokens = self.input_tokens + + output_tokens: int | None | Unset + if isinstance(self.output_tokens, Unset): + output_tokens = UNSET + else: + output_tokens = self.output_tokens + + total_tokens: int | None | Unset + if isinstance(self.total_tokens, Unset): + total_tokens = UNSET + else: + total_tokens = self.total_tokens + + cost: float | None | Unset + if isinstance(self.cost, Unset): + cost = UNSET + else: + cost = self.cost + + cached_tokens: int | None | Unset + if isinstance(self.cached_tokens, Unset): + cached_tokens = UNSET + else: + cached_tokens = self.cached_tokens + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if input_tokens is not UNSET: + field_dict["input_tokens"] = input_tokens + if output_tokens is not UNSET: + field_dict["output_tokens"] = output_tokens + if total_tokens is not UNSET: + field_dict["total_tokens"] = total_tokens + if cost is not UNSET: + field_dict["cost"] = cost + if cached_tokens is not UNSET: + field_dict["cached_tokens"] = cached_tokens + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + + def _parse_input_tokens(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + input_tokens = _parse_input_tokens(d.pop("input_tokens", UNSET)) + + def _parse_output_tokens(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + output_tokens = _parse_output_tokens(d.pop("output_tokens", UNSET)) + + def _parse_total_tokens(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + total_tokens = _parse_total_tokens(d.pop("total_tokens", UNSET)) + + def _parse_cost(data: object) -> float | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(float | None | Unset, data) + + cost = _parse_cost(d.pop("cost", UNSET)) + + def _parse_cached_tokens(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + cached_tokens = _parse_cached_tokens(d.pop("cached_tokens", UNSET)) + + message_usage = cls( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=total_tokens, + cost=cost, + cached_tokens=cached_tokens, + ) + + message_usage.additional_properties = d + return message_usage + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/synthetic_user_case.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/synthetic_user_case.py new file mode 100644 index 0000000000..2940f82840 --- /dev/null +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/synthetic_user_case.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="SyntheticUserCase") + + +@_attrs_define +class SyntheticUserCase: + """One generated synthetic-user case used to seed a probing conversation. + + Attributes: + seed_prompt (str): Synthetic user's first message, written in their own voice. + synthetic_user_info (str): XML-tagged blob describing the synthetic user, in the format + `.........`. Parse client-side. Tag names + are stable; future versions may add new optional tags, but existing tags will not be renamed or removed. + """ + + seed_prompt: str + synthetic_user_info: str + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + seed_prompt = self.seed_prompt + + synthetic_user_info = self.synthetic_user_info + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "seed_prompt": seed_prompt, + "synthetic_user_info": synthetic_user_info, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + seed_prompt = d.pop("seed_prompt") + + synthetic_user_info = d.pop("synthetic_user_info") + + synthetic_user_case = cls( + seed_prompt=seed_prompt, + synthetic_user_info=synthetic_user_info, + ) + + synthetic_user_case.additional_properties = d + return synthetic_user_case + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output.py index 5ad895f12c..d00a4b5397 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output.py @@ -6,7 +6,6 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -from dateutil.parser import isoparse from ..types import UNSET, Unset @@ -153,7 +152,7 @@ def _parse_path(data: object) -> None | str | Unset: if isinstance(_created_at, Unset): created_at = UNSET else: - created_at = isoparse(_created_at) + created_at = datetime.datetime.fromisoformat(_created_at) created_by = d.pop("created_by", UNSET) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output_rating.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output_rating.py index c8fcad6f53..787a4a5a2e 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output_rating.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_output_rating.py @@ -6,7 +6,6 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -from dateutil.parser import isoparse from ..models.task_output_rating_type import TaskOutputRatingType from ..types import UNSET, Unset @@ -151,7 +150,7 @@ def _parse_path(data: object) -> None | str | Unset: if isinstance(_created_at, Unset): created_at = UNSET else: - created_at = isoparse(_created_at) + created_at = datetime.datetime.fromisoformat(_created_at) created_by = d.pop("created_by", UNSET) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_run.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_run.py index 51913716fe..25baeb286c 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_run.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/task_run.py @@ -6,7 +6,6 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field -from dateutil.parser import isoparse from ..types import UNSET, Unset @@ -18,6 +17,7 @@ from ..models.chat_completion_tool_message_param_wrapper import ChatCompletionToolMessageParamWrapper from ..models.chat_completion_user_message_param import ChatCompletionUserMessageParam from ..models.data_source import DataSource + from ..models.message_usage import MessageUsage from ..models.task_output import TaskOutput from ..models.task_run_intermediate_outputs_type_0 import TaskRunIntermediateOutputsType0 from ..models.usage import Usage @@ -59,6 +59,9 @@ class TaskRun: reporting. usage (None | Unset | Usage): Usage information for the task run. This includes the number of input tokens, output tokens, and total tokens used. + cumulative_usage (MessageUsage | None | Unset): Sum of per-message token usage and cost across the entire trace, + including any seeded prior trace. None on records created before this field existed. For a fresh (non-seeded) + run, the token / cost fields equal those of `usage`. trace (list[ChatCompletionAssistantMessageParamWrapper | ChatCompletionDeveloperMessageParam | ChatCompletionFunctionMessageParam | ChatCompletionSystemMessageParam | ChatCompletionToolMessageParamWrapper | ChatCompletionUserMessageParam] | None | Unset): The trace of the task run in OpenAI format. This is the list of @@ -81,6 +84,7 @@ class TaskRun: intermediate_outputs: None | TaskRunIntermediateOutputsType0 | Unset = UNSET tags: list[str] | Unset = UNSET usage: None | Unset | Usage = UNSET + cumulative_usage: MessageUsage | None | Unset = UNSET trace: ( list[ ChatCompletionAssistantMessageParamWrapper @@ -103,6 +107,7 @@ def to_dict(self) -> dict[str, Any]: from ..models.chat_completion_tool_message_param_wrapper import ChatCompletionToolMessageParamWrapper from ..models.chat_completion_user_message_param import ChatCompletionUserMessageParam from ..models.data_source import DataSource + from ..models.message_usage import MessageUsage from ..models.task_output import TaskOutput from ..models.task_run_intermediate_outputs_type_0 import TaskRunIntermediateOutputsType0 from ..models.usage import Usage @@ -175,6 +180,14 @@ def to_dict(self) -> dict[str, Any]: else: usage = self.usage + cumulative_usage: dict[str, Any] | None | Unset + if isinstance(self.cumulative_usage, Unset): + cumulative_usage = UNSET + elif isinstance(self.cumulative_usage, MessageUsage): + cumulative_usage = self.cumulative_usage.to_dict() + else: + cumulative_usage = self.cumulative_usage + trace: list[dict[str, Any]] | None | Unset if isinstance(self.trace, Unset): trace = UNSET @@ -237,6 +250,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["tags"] = tags if usage is not UNSET: field_dict["usage"] = usage + if cumulative_usage is not UNSET: + field_dict["cumulative_usage"] = cumulative_usage if trace is not UNSET: field_dict["trace"] = trace if parent_task_run_id is not UNSET: @@ -253,6 +268,7 @@ def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: from ..models.chat_completion_tool_message_param_wrapper import ChatCompletionToolMessageParamWrapper from ..models.chat_completion_user_message_param import ChatCompletionUserMessageParam from ..models.data_source import DataSource + from ..models.message_usage import MessageUsage from ..models.task_output import TaskOutput from ..models.task_run_intermediate_outputs_type_0 import TaskRunIntermediateOutputsType0 from ..models.usage import Usage @@ -289,7 +305,7 @@ def _parse_path(data: object) -> None | str | Unset: if isinstance(_created_at, Unset): created_at = UNSET else: - created_at = isoparse(_created_at) + created_at = datetime.datetime.fromisoformat(_created_at) created_by = d.pop("created_by", UNSET) @@ -372,6 +388,23 @@ def _parse_usage(data: object) -> None | Unset | Usage: usage = _parse_usage(d.pop("usage", UNSET)) + def _parse_cumulative_usage(data: object) -> MessageUsage | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + cumulative_usage_type_0 = MessageUsage.from_dict(data) + + return cumulative_usage_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(MessageUsage | None | Unset, data) + + cumulative_usage = _parse_cumulative_usage(d.pop("cumulative_usage", UNSET)) + def _parse_trace( data: object, ) -> ( @@ -500,6 +533,7 @@ def _parse_parent_task_run_id(data: object) -> None | str | Unset: intermediate_outputs=intermediate_outputs, tags=tags, usage=usage, + cumulative_usage=cumulative_usage, trace=trace, parent_task_run_id=parent_task_run_id, ) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/usage.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/usage.py index 970b2578e9..68e1de6951 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/usage.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/usage.py @@ -13,15 +13,22 @@ @_attrs_define class Usage: - """Token usage and cost information for a task run. - - Attributes: - input_tokens (int | None | Unset): The number of input tokens used in the task run. - output_tokens (int | None | Unset): The number of output tokens used in the task run. - total_tokens (int | None | Unset): The total number of tokens used in the task run. - cost (float | None | Unset): The cost of the task run in US dollars, saved at runtime (prices can change over - time). - cached_tokens (int | None | Unset): Number of tokens served from prompt cache. None if not reported. + """Token usage, cost, and aggregate LLM latency for a per-run accumulator. + + Extends :class:`MessageUsage` with ``total_llm_latency_ms``, which is + only meaningful while a single run is in flight (its model calls run + sequentially in real time). For per-message records and full-trace + sums use :class:`MessageUsage` — those values would mix latencies + from different points in time, so the field doesn't apply. + + Attributes: + input_tokens (int | None | Unset): The number of input tokens used. + output_tokens (int | None | Unset): The number of output tokens used. + total_tokens (int | None | Unset): The total number of tokens used. + cost (float | None | Unset): The cost in US dollars, saved at runtime (prices can change over time). + cached_tokens (int | None | Unset): Number of tokens served from prompt cache. None if not reported. + total_llm_latency_ms (int | None | Unset): Total time spent waiting on LLM API calls in milliseconds. Sum of + per-call latencies, excludes tool execution time. """ input_tokens: int | None | Unset = UNSET @@ -29,6 +36,7 @@ class Usage: total_tokens: int | None | Unset = UNSET cost: float | None | Unset = UNSET cached_tokens: int | None | Unset = UNSET + total_llm_latency_ms: int | None | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -62,6 +70,12 @@ def to_dict(self) -> dict[str, Any]: else: cached_tokens = self.cached_tokens + total_llm_latency_ms: int | None | Unset + if isinstance(self.total_llm_latency_ms, Unset): + total_llm_latency_ms = UNSET + else: + total_llm_latency_ms = self.total_llm_latency_ms + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -75,6 +89,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["cost"] = cost if cached_tokens is not UNSET: field_dict["cached_tokens"] = cached_tokens + if total_llm_latency_ms is not UNSET: + field_dict["total_llm_latency_ms"] = total_llm_latency_ms return field_dict @@ -127,12 +143,22 @@ def _parse_cached_tokens(data: object) -> int | None | Unset: cached_tokens = _parse_cached_tokens(d.pop("cached_tokens", UNSET)) + def _parse_total_llm_latency_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + total_llm_latency_ms = _parse_total_llm_latency_ms(d.pop("total_llm_latency_ms", UNSET)) + usage = cls( input_tokens=input_tokens, output_tokens=output_tokens, total_tokens=total_tokens, cost=cost, cached_tokens=cached_tokens, + total_llm_latency_ms=total_llm_latency_ms, ) usage.additional_properties = d From c122022627ba1b0c4bb299da524b3873a51c6ec1 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:07:57 -0700 Subject: [PATCH 02/36] feat(libs/core): add synthetic_user player module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OSS-side per-turn synthetic-user invocation — the replacement for kiln_server's removed /respond endpoint. Lives in libs/core/kiln_ai/synthetic_user/ so the runner can call the LLM using the user's own provider keys rather than a hosted endpoint. Modules: - models — Pydantic SyntheticUserInfo (parsed form) + SyntheticUserDriverConfig. - parser — tagged-blob ↔ SyntheticUserInfo. Required: , ; optional: . Unknown tags ignored (forward-compat). - role_swap — flips eval-frame user/assistant labels into LLM-frame labels; raises on system/tool roles (the driver filters those upstream) and on non-string content. - prompt — persona-playing system prompt. No / guidance: drive loop is fixed-length; SU stays engaged across the conversation. - driver — SyntheticUserDriver. Parses the blob once at construction, renders the system prompt once, builds the adapter once. respond() filters visible roles, role-swaps, prepends the system prompt as prior_trace[0], calls adapter.invoke_returning_run_output (in-memory — the SU never persists a TaskRun), returns the raw string. 56 unit tests covering: parser roundtrip / required-tag enforcement / whitespace / unknown-tag forward-compat; role_swap empty/alternating/ preserves-order/raises-on-system-or-tool; prompt structural assertions (persona/goal/conventions present, behavior_guidance only when set, no /); driver happy path, role-swap shape, custom visible_roles, ends-on-assistant invariant, non-string output guard, parse-error on construction, adapter reuse across turns. Co-Authored-By: Claude Opus 4.7 (1M context) --- libs/core/kiln_ai/synthetic_user/__init__.py | 37 +++ libs/core/kiln_ai/synthetic_user/driver.py | 128 ++++++++ libs/core/kiln_ai/synthetic_user/models.py | 48 +++ libs/core/kiln_ai/synthetic_user/parser.py | 60 ++++ libs/core/kiln_ai/synthetic_user/prompt.py | 34 +++ libs/core/kiln_ai/synthetic_user/role_swap.py | 61 ++++ .../kiln_ai/synthetic_user/test_driver.py | 278 ++++++++++++++++++ .../kiln_ai/synthetic_user/test_models.py | 69 +++++ .../kiln_ai/synthetic_user/test_parser.py | 145 +++++++++ .../kiln_ai/synthetic_user/test_prompt.py | 95 ++++++ .../kiln_ai/synthetic_user/test_role_swap.py | 77 +++++ 11 files changed, 1032 insertions(+) create mode 100644 libs/core/kiln_ai/synthetic_user/__init__.py create mode 100644 libs/core/kiln_ai/synthetic_user/driver.py create mode 100644 libs/core/kiln_ai/synthetic_user/models.py create mode 100644 libs/core/kiln_ai/synthetic_user/parser.py create mode 100644 libs/core/kiln_ai/synthetic_user/prompt.py create mode 100644 libs/core/kiln_ai/synthetic_user/role_swap.py create mode 100644 libs/core/kiln_ai/synthetic_user/test_driver.py create mode 100644 libs/core/kiln_ai/synthetic_user/test_models.py create mode 100644 libs/core/kiln_ai/synthetic_user/test_parser.py create mode 100644 libs/core/kiln_ai/synthetic_user/test_prompt.py create mode 100644 libs/core/kiln_ai/synthetic_user/test_role_swap.py diff --git a/libs/core/kiln_ai/synthetic_user/__init__.py b/libs/core/kiln_ai/synthetic_user/__init__.py new file mode 100644 index 0000000000..7676c14f60 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/__init__.py @@ -0,0 +1,37 @@ +"""Local synthetic-user player. + +OSS-side per-turn invocation: replaces the removed kiln_server `/respond` +HTTP endpoint. Calls the LLM using the user's own provider keys. + +Public surface: + +- `SyntheticUserDriver` — construct once per case, call `respond()` per turn. +- `SyntheticUserInfo` / `SyntheticUserDriverConfig` — typed configs. +- `parse_synthetic_user_info` / `build_synthetic_user_info` — tagged-blob codec. +- `SyntheticUserInfoParseError` — raised on malformed blob. +- `role_swap` — exposed for callers that drive the loop themselves. +""" + +from kiln_ai.synthetic_user.driver import SyntheticUserDriver +from kiln_ai.synthetic_user.models import ( + SyntheticUserDriverConfig, + SyntheticUserInfo, + VisibleMessageRole, +) +from kiln_ai.synthetic_user.parser import ( + SyntheticUserInfoParseError, + build_synthetic_user_info, + parse_synthetic_user_info, +) +from kiln_ai.synthetic_user.role_swap import role_swap + +__all__ = [ + "SyntheticUserDriver", + "SyntheticUserDriverConfig", + "SyntheticUserInfo", + "SyntheticUserInfoParseError", + "VisibleMessageRole", + "build_synthetic_user_info", + "parse_synthetic_user_info", + "role_swap", +] diff --git a/libs/core/kiln_ai/synthetic_user/driver.py b/libs/core/kiln_ai/synthetic_user/driver.py new file mode 100644 index 0000000000..dee2625d79 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/driver.py @@ -0,0 +1,128 @@ +"""Per-turn synthetic-user driver — the OSS-side replacement for kiln_server's +removed `/respond` route. + +Wraps a kiln_ai LiteLLM adapter and exposes a single async `respond()` that: +1. Filters the eval-frame conversation to `visible_message_roles`. +2. Role-swaps user/assistant so the LLM is generating the SU's reply. +3. Calls the adapter with the persona system prompt prepended as + `prior_trace` and the latest swapped user turn as `input`. + +The driver does NOT persist `TaskRun`s — it uses +`adapter.invoke_returning_run_output(...)` which builds an in-memory run +without writing to disk. The eval-dataset chain consists only of *target* +TaskRuns (created elsewhere by `adapter.invoke(...)`). The SU is an +orchestration component, not an eval-dataset row. +""" + +from kiln_ai.adapters.adapter_registry import adapter_for_task +from kiln_ai.datamodel.datamodel_enums import StructuredOutputMode +from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig +from kiln_ai.datamodel.task import Task +from kiln_ai.synthetic_user.models import ( + SyntheticUserDriverConfig, + SyntheticUserInfo, +) +from kiln_ai.synthetic_user.parser import parse_synthetic_user_info +from kiln_ai.synthetic_user.prompt import render_system_prompt +from kiln_ai.synthetic_user.role_swap import role_swap +from kiln_ai.utils.open_ai_types import ( + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, +) + + +class SyntheticUserDriver: + """Plays one synthetic user across multiple turns. + + Constructed once per case from the case's tagged blob + driver config; + `respond(conversation)` is called once per turn. The adapter is built + at construction time and reused across all turns of the case. + """ + + def __init__( + self, + synthetic_user_info_blob: str, + driver_config: SyntheticUserDriverConfig, + ): + # Parse the blob once at construction — fail fast on a malformed case. + self._info: SyntheticUserInfo = parse_synthetic_user_info( + synthetic_user_info_blob + ) + self._driver_config = driver_config + self._system_prompt: str = render_system_prompt(self._info) + + # In-memory Task; nothing is persisted. The persona-playing system + # prompt rides on `prior_trace[0]` each call, so this `instruction` + # is effectively unused at runtime — kiln_ai's MultiturnFormatter + # uses the first system message in `prior_trace` and skips the + # task's instruction when `prior_trace` is non-empty. The Task + # model requires a non-empty instruction, hence the placeholder. + self._task = Task( + name="synthetic_user_driver", + description="In-memory SU player. Not persisted.", + instruction=( + "Placeholder — the persona-playing system prompt is supplied " + "via prior_trace on every adapter call." + ), + ) + # Same RunConfigProperties shape used elsewhere; `structured_output_mode` + # is `default` because SU output is free text (no JSON schema). + self._run_config = KilnAgentRunConfigProperties( + model_name=driver_config.model_name, + model_provider_name=driver_config.model_provider_name, + prompt_id="simple_prompt_builder", + structured_output_mode=StructuredOutputMode.default, + tools_config=ToolsRunConfig(tools=[]), + ) + self._adapter = adapter_for_task(self._task, self._run_config) + + async def respond(self, conversation: list[ChatCompletionMessageParam]) -> str: + """Return the SU's next message. + + `conversation` is in the eval frame and must end on an `assistant` + (target) turn — the SU is responding to that turn. Drive-loop + termination is the caller's concern; this just produces one reply. + """ + # 1) Filter to visible roles (drop system/tool if present). + visible = [ + m + for m in conversation + if m["role"] in self._driver_config.visible_message_roles + ] + # 2) Invariants this driver enforces (moved from kiln_server's + # removed /respond route validator). + if not visible: + raise ValueError("No LLM-visible messages in conversation.") + if visible[-1]["role"] != "assistant": + raise ValueError( + "Conversation must end on an assistant (target) turn — the SU " + "is responding to that turn." + ) + + # 3) Role-swap then assemble prior_trace + input. The last swapped + # turn becomes the LLM `input`; everything before it goes into + # `prior_trace` with a system message prepended. + swapped = role_swap(visible) + last = swapped[-1] + user_input = last["content"] + if not isinstance(user_input, str): + raise RuntimeError( + "synthetic user input must be a plain string after role_swap" + ) + + system_msg: ChatCompletionSystemMessageParam = { + "role": "system", + "content": self._system_prompt, + } + prior_trace: list[ChatCompletionMessageParam] = [system_msg, *swapped[:-1]] + + # 4) Adapter call. invoke_returning_run_output returns + # (TaskRun, RunOutput) without writing the TaskRun to disk. + _task_run, run_output = await self._adapter.invoke_returning_run_output( + user_input, prior_trace=prior_trace + ) + raw = run_output.output + if not isinstance(raw, str): + raise RuntimeError("synthetic user returned non-string output") + + return raw diff --git a/libs/core/kiln_ai/synthetic_user/models.py b/libs/core/kiln_ai/synthetic_user/models.py new file mode 100644 index 0000000000..e532a7fe3b --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/models.py @@ -0,0 +1,48 @@ +"""Internal models for the synthetic-user player. + +`SyntheticUserInfo` is the parsed form of the tagged blob the server sends +on `/generate`; the parser produces it and `prompt.render_system_prompt` +consumes it. `SyntheticUserDriverConfig` carries the per-eval runtime +config — model, provider, role visibility — moved here from the kiln_server +wire (where it used to live on the now-deleted `/respond` request). +""" + +from typing import Literal + +from pydantic import BaseModel, Field + +from kiln_ai.datamodel.datamodel_enums import ModelProviderName + +VisibleMessageRole = Literal["user", "assistant"] + + +class SyntheticUserInfo(BaseModel): + """Parsed form of the tagged synthetic_user_info blob. + + Built by `parser.parse_synthetic_user_info` from the wire string. + Used by `prompt.render_system_prompt` to assemble the per-request + system prompt. + + Extend with new fields as the server-side generator emits new tags — + the parser ignores unknown tags so this is forward-compat by default. + """ + + persona: str + goal: str + behavior_guidance: str | None = None + + +class SyntheticUserDriverConfig(BaseModel): + """Per-eval runtime config for the SU's LLM driver. + + No `temperature` field — runs at the chosen model's default. The driver + intentionally does not own temperature: the persona-playing prompt and + `behavior_guidance` carry style; temperature is a model-level concern + surfaced elsewhere when it matters. + """ + + model_name: str + model_provider_name: ModelProviderName + visible_message_roles: list[VisibleMessageRole] = Field( + default_factory=lambda: ["user", "assistant"] + ) diff --git a/libs/core/kiln_ai/synthetic_user/parser.py b/libs/core/kiln_ai/synthetic_user/parser.py new file mode 100644 index 0000000000..0d40c7b4d4 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/parser.py @@ -0,0 +1,60 @@ +"""Parse / build the tagged synthetic_user_info blob. + +Server-side this is the wire format; OSS-side this is the on-storage form. +Both ends call these helpers — single source of truth for the tag schema. + +Tag rules: +- Greedy first match per known tag; nested tags not supported. +- Whitespace trimmed from each tag's content. +- Unknown tags are ignored (forward-compat for future generator output). +- `` and `` are required; parse raises if missing or empty. +- `` is optional; missing → that section is omitted + from the rendered system prompt. +""" + +import re + +from kiln_ai.synthetic_user.models import SyntheticUserInfo + + +class SyntheticUserInfoParseError(ValueError): + """The blob is missing a required tag, or all required tags are empty.""" + + +def _extract(blob: str, tag: str) -> str | None: + """Greedy first-match. Returns the trimmed content, or None if no match.""" + m = re.search(rf"<{tag}>(.*?)", blob, re.DOTALL) + if m is None: + return None + return m.group(1).strip() + + +def parse_synthetic_user_info(blob: str) -> SyntheticUserInfo: + """Parse the tagged blob. Strict on required tags; lenient on optional.""" + persona = _extract(blob, "persona") + if not persona: + raise SyntheticUserInfoParseError( + "Missing or empty required tag in synthetic_user_info blob." + ) + goal = _extract(blob, "goal") + if not goal: + raise SyntheticUserInfoParseError( + "Missing or empty required tag in synthetic_user_info blob." + ) + behavior_guidance = _extract(blob, "behavior_guidance") or None + return SyntheticUserInfo( + persona=persona, + goal=goal, + behavior_guidance=behavior_guidance, + ) + + +def build_synthetic_user_info(info: SyntheticUserInfo) -> str: + """Inverse of parse — for tests and any callers that need to construct a blob.""" + parts = [ + f"{info.persona}", + f"{info.goal}", + ] + if info.behavior_guidance: + parts.append(f"{info.behavior_guidance}") + return "".join(parts) diff --git a/libs/core/kiln_ai/synthetic_user/prompt.py b/libs/core/kiln_ai/synthetic_user/prompt.py new file mode 100644 index 0000000000..f9d0521d56 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/prompt.py @@ -0,0 +1,34 @@ +"""Persona-playing system prompt rendered per request for the synthetic user. + +The prompt is built once per case (when the driver is constructed) and +reused across all turns. No early-termination guidance — the drive loop +runs for a fixed number of turns; the SU is told to stay engaged across +the whole conversation rather than try to wrap up early. +""" + +from kiln_ai.synthetic_user.models import SyntheticUserInfo + +_OPENING = ( + "You are playing the role of a user interacting with an AI assistant. " + "Stay in character — respond as the user would, not as the assistant." +) + +_CONVENTIONS = """## Conversation style +- Reply naturally and conversationally, as a real user would. +- React to what the assistant actually said in their last message. +- Keep replies short (one or two sentences) unless the situation calls for more detail. +- Do not narrate your own behavior ("As a user, I will now…"). Speak as the user. +- Stay engaged across the whole conversation. The drive loop runs for a fixed number of turns; do not try to wrap up, conclude, or end the conversation early. Keep asking follow-ups, varying approach as needed.""" + + +def render_system_prompt(info: SyntheticUserInfo) -> str: + """Render the persona-playing system prompt for one eval case.""" + sections = [ + _OPENING, + f"## Your persona\n{info.persona}", + f"## Your goal in this conversation\n{info.goal}", + ] + if info.behavior_guidance: + sections.append(f"## How you behave\n{info.behavior_guidance}") + sections.append(_CONVENTIONS) + return "\n\n".join(sections) diff --git a/libs/core/kiln_ai/synthetic_user/role_swap.py b/libs/core/kiln_ai/synthetic_user/role_swap.py new file mode 100644 index 0000000000..2ee60e66ec --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/role_swap.py @@ -0,0 +1,61 @@ +"""Role-swap eval-frame conversation roles into LLM-frame roles. + +Chat models are trained to generate `assistant`-labeled responses, so we +flip the eval-frame labels before the call: the LLM's "assistant" turn IS +the synthetic user. + + Eval frame: LLM frame (post-swap): + user = synthetic user assistant = synthetic user + assistant = target agent user = target agent + +The driver filters `visible_message_roles` upstream, so a system or tool +turn reaching here is an internal invariant violation — fail loud rather +than silently drop. +""" + +from kiln_ai.utils.open_ai_types import ( + ChatCompletionAssistantMessageParamWrapper, + ChatCompletionMessageParam, + ChatCompletionUserMessageParam, +) + + +def role_swap( + conversation: list[ChatCompletionMessageParam], +) -> list[ChatCompletionMessageParam]: + """Flip eval-frame user/assistant labels into LLM-frame labels. + + Only `user` and `assistant` are handled. The driver is expected to + have filtered out other roles before calling this. + """ + result: list[ChatCompletionMessageParam] = [] + for msg in conversation: + role = msg["role"] + if role not in ("user", "assistant"): + raise ValueError( + f"role_swap received unsupported role {role!r}; " + "the driver should have filtered it" + ) + # The TypedDict union allows non-string content for multimodal / + # tool turns, but the synthetic user only ever sees plain text from + # the target. Narrowing here lets us assign into the swapped wrapper + # type without a cast. + content = msg["content"] + if not isinstance(content, str): + raise ValueError( + f"role_swap requires string content for role {role!r}; " + f"got {type(content).__name__}" + ) + if role == "user": + assistant_msg: ChatCompletionAssistantMessageParamWrapper = { + "role": "assistant", + "content": content, + } + result.append(assistant_msg) + else: # role == "assistant" + user_msg: ChatCompletionUserMessageParam = { + "role": "user", + "content": content, + } + result.append(user_msg) + return result diff --git a/libs/core/kiln_ai/synthetic_user/test_driver.py b/libs/core/kiln_ai/synthetic_user/test_driver.py new file mode 100644 index 0000000000..754ebcceb9 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_driver.py @@ -0,0 +1,278 @@ +"""Unit tests for SyntheticUserDriver. + +`adapter_for_task` is monkeypatched to return a fake adapter whose +`invoke_returning_run_output` is an AsyncMock — no real LLM calls. Tests +focus on the driver's per-turn responsibilities: input shaping +(filtering, role swap, system prompt prepend) and output validation. +""" + +from unittest.mock import AsyncMock, Mock + +import pytest + +from kiln_ai.adapters.run_output import RunOutput +from kiln_ai.datamodel.datamodel_enums import ModelProviderName +from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user import driver as driver_mod +from kiln_ai.synthetic_user.driver import SyntheticUserDriver +from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig +from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError +from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam + +_BLOB = ( + "A 30-something professional" + "Find the 2016 RRSP limit" + "Press for specifics" +) + +_DRIVER_CONFIG = SyntheticUserDriverConfig( + model_name="claude_4_5_haiku", + model_provider_name=ModelProviderName.openrouter, +) + + +def _fake_run_output(text: str | dict = "hi from the SU") -> RunOutput: + return RunOutput(output=text, intermediate_outputs=None) + + +def _patch_adapter(monkeypatch: pytest.MonkeyPatch, return_value: RunOutput) -> Mock: + """Replace adapter_for_task with a stub returning a mock adapter whose + invoke_returning_run_output yields (Mock(spec=TaskRun), return_value). + Returns the mock adapter so tests can assert call args. + """ + adapter = Mock() + adapter.invoke_returning_run_output = AsyncMock( + return_value=(Mock(spec=TaskRun), return_value) + ) + monkeypatch.setattr( + driver_mod, "adapter_for_task", lambda task, run_config: adapter + ) + return adapter + + +# ───────────────────────── construction ───────────────────────── + + +def test_construction_parses_blob_and_stores_info( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + assert drv._info.persona == "A 30-something professional" + assert drv._info.goal == "Find the 2016 RRSP limit" + assert drv._info.behavior_guidance == "Press for specifics" + + +def test_construction_renders_system_prompt_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + # System prompt is rendered on construction and reused. + assert "A 30-something professional" in drv._system_prompt + assert "Find the 2016 RRSP limit" in drv._system_prompt + assert "Press for specifics" in drv._system_prompt + + +def test_construction_raises_on_malformed_blob(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + with pytest.raises(SyntheticUserInfoParseError): + SyntheticUserDriver("no tags here at all", _DRIVER_CONFIG) + + +def test_construction_raises_on_empty_required_tag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + with pytest.raises(SyntheticUserInfoParseError): + SyntheticUserDriver("G", _DRIVER_CONFIG) + + +# ───────────────────────── respond — happy path ───────────────────────── + + +@pytest.mark.asyncio +async def test_respond_returns_adapter_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter = _patch_adapter(monkeypatch, _fake_run_output("the SU's reply")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + + out = await drv.respond(conversation) + + assert out == "the SU's reply" + adapter.invoke_returning_run_output.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_respond_role_swaps_and_prepends_system_prompt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """End-to-end shape check: input is the role-swapped LAST user turn; + prior_trace is [system, ...role-swapped earlier turns]. + """ + adapter = _patch_adapter(monkeypatch, _fake_run_output("ok")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "seed-user-1"}, + {"role": "assistant", "content": "target-reply-1"}, + {"role": "user", "content": "seed-user-2"}, + {"role": "assistant", "content": "target-reply-2"}, + ] + + await drv.respond(conversation) + + call = adapter.invoke_returning_run_output.await_args + # `input` is the role-swapped LAST eval-frame turn — was "assistant" with + # content "target-reply-2", becomes "user" content "target-reply-2". + positional = call.args + keyword = call.kwargs + # invoke_returning_run_output(user_input, prior_trace=...) + assert positional[0] == "target-reply-2" + + prior_trace = keyword["prior_trace"] + # prior_trace = [system, role-swapped turns 0..-2] + assert prior_trace[0]["role"] == "system" + assert "A 30-something professional" in prior_trace[0]["content"] + # The three earlier turns, all role-swapped. + assert [m["role"] for m in prior_trace[1:]] == ["assistant", "user", "assistant"] + assert prior_trace[1]["content"] == "seed-user-1" + assert prior_trace[2]["content"] == "target-reply-1" + assert prior_trace[3]["content"] == "seed-user-2" + + +@pytest.mark.asyncio +async def test_respond_filters_visible_message_roles( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A system turn in the conversation must be dropped before role-swap.""" + adapter = _patch_adapter(monkeypatch, _fake_run_output("ok")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "system", "content": "should be dropped"}, + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + + await drv.respond(conversation) + + call = adapter.invoke_returning_run_output.await_args + prior_trace = call.kwargs["prior_trace"] + # prior_trace = [our system prompt, role-swapped user from u1] + # The "system" eval-frame turn is filtered out. + assert len(prior_trace) == 2 + assert prior_trace[0]["content"] == drv._system_prompt + assert prior_trace[1] == {"role": "assistant", "content": "u1"} + + +@pytest.mark.asyncio +async def test_respond_with_custom_visible_roles( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Custom visibility set is honored — the driver doesn't hardcode the default.""" + adapter = _patch_adapter(monkeypatch, _fake_run_output("ok")) + drv = SyntheticUserDriver( + _BLOB, + SyntheticUserDriverConfig( + model_name="x", + model_provider_name=ModelProviderName.openrouter, + visible_message_roles=["assistant"], + ), + ) + # Only assistant turns visible — the user turn gets filtered out, leaving + # only the assistant for /respond. After filter, the conversation is a + # single "assistant" message, which IS the required ends-on-assistant + # shape: visible=[asst]; swap→[user]; last is input; prior_trace=[sys]. + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + + await drv.respond(conversation) + + call = adapter.invoke_returning_run_output.await_args + assert call.args[0] == "a1" + assert len(call.kwargs["prior_trace"]) == 1 # just the system prompt + + +# ───────────────────────── respond — invariants ───────────────────────── + + +@pytest.mark.asyncio +async def test_respond_raises_when_no_visible_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + # All messages filtered out by visible_message_roles. + conversation: list[ChatCompletionMessageParam] = [ + {"role": "system", "content": "sys"}, + ] + with pytest.raises(ValueError, match="No LLM-visible"): + await drv.respond(conversation) + + +@pytest.mark.asyncio +async def test_respond_raises_when_empty_conversation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + with pytest.raises(ValueError, match="No LLM-visible"): + await drv.respond([]) + + +@pytest.mark.asyncio +async def test_respond_raises_when_ends_on_user( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_adapter(monkeypatch, _fake_run_output()) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u1"}, + ] + with pytest.raises(ValueError, match="end on an assistant"): + await drv.respond(conversation) + + +@pytest.mark.asyncio +async def test_respond_raises_on_non_string_adapter_output( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """RunOutput.output is typed as Dict | str — only str is valid for the SU.""" + _patch_adapter(monkeypatch, _fake_run_output({"unexpected": "structured output"})) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + with pytest.raises(RuntimeError, match="non-string output"): + await drv.respond(conversation) + + +# ───────────────────────── respond — reuse ───────────────────────── + + +@pytest.mark.asyncio +async def test_respond_reuses_adapter_across_turns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The adapter is built once in the constructor and reused per turn.""" + adapter = _patch_adapter(monkeypatch, _fake_run_output("reply")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + await drv.respond(conversation) + await drv.respond(conversation) + await drv.respond(conversation) + + assert adapter.invoke_returning_run_output.await_count == 3 + # The driver's _adapter reference doesn't change between calls. diff --git a/libs/core/kiln_ai/synthetic_user/test_models.py b/libs/core/kiln_ai/synthetic_user/test_models.py new file mode 100644 index 0000000000..1bcc9413cb --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_models.py @@ -0,0 +1,69 @@ +"""Unit tests for SyntheticUserInfo / SyntheticUserDriverConfig.""" + +import pytest +from pydantic import ValidationError + +from kiln_ai.datamodel.datamodel_enums import ModelProviderName +from kiln_ai.synthetic_user.models import ( + SyntheticUserDriverConfig, + SyntheticUserInfo, +) + + +def test_synthetic_user_info_required_fields() -> None: + info = SyntheticUserInfo(persona="p", goal="g") + assert info.persona == "p" + assert info.goal == "g" + assert info.behavior_guidance is None + + +def test_synthetic_user_info_accepts_behavior_guidance() -> None: + info = SyntheticUserInfo(persona="p", goal="g", behavior_guidance="b") + assert info.behavior_guidance == "b" + + +def test_synthetic_user_info_rejects_missing_persona() -> None: + with pytest.raises(ValidationError): + SyntheticUserInfo(goal="g") # type: ignore[call-arg] + + +def test_synthetic_user_info_rejects_missing_goal() -> None: + with pytest.raises(ValidationError): + SyntheticUserInfo(persona="p") # type: ignore[call-arg] + + +def test_driver_config_default_visible_roles() -> None: + cfg = SyntheticUserDriverConfig( + model_name="x", model_provider_name=ModelProviderName.openrouter + ) + assert cfg.visible_message_roles == ["user", "assistant"] + + +def test_driver_config_default_is_per_instance() -> None: + # Pydantic v2 + default_factory must not share the list across instances. + a = SyntheticUserDriverConfig( + model_name="x", model_provider_name=ModelProviderName.openrouter + ) + b = SyntheticUserDriverConfig( + model_name="y", model_provider_name=ModelProviderName.openrouter + ) + a.visible_message_roles.append("user") # type: ignore[arg-type] + assert b.visible_message_roles == ["user", "assistant"] + + +def test_driver_config_accepts_explicit_visible_roles() -> None: + cfg = SyntheticUserDriverConfig( + model_name="x", + model_provider_name=ModelProviderName.openrouter, + visible_message_roles=["assistant"], + ) + assert cfg.visible_message_roles == ["assistant"] + + +def test_driver_config_rejects_invalid_role_literal() -> None: + with pytest.raises(ValidationError): + SyntheticUserDriverConfig( + model_name="x", + model_provider_name=ModelProviderName.openrouter, + visible_message_roles=["system"], # type: ignore[list-item] + ) diff --git a/libs/core/kiln_ai/synthetic_user/test_parser.py b/libs/core/kiln_ai/synthetic_user/test_parser.py new file mode 100644 index 0000000000..b392180f80 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_parser.py @@ -0,0 +1,145 @@ +"""Unit tests for the tagged blob parser/builder.""" + +import pytest + +from kiln_ai.synthetic_user.models import SyntheticUserInfo +from kiln_ai.synthetic_user.parser import ( + SyntheticUserInfoParseError, + build_synthetic_user_info, + parse_synthetic_user_info, +) + +# ───────────────────────── parse ───────────────────────── + + +def test_parse_all_three_tags() -> None: + blob = "PGB" + info = parse_synthetic_user_info(blob) + assert info.persona == "P" + assert info.goal == "G" + assert info.behavior_guidance == "B" + + +def test_parse_only_required_tags() -> None: + blob = "PG" + info = parse_synthetic_user_info(blob) + assert info.behavior_guidance is None + + +def test_parse_strips_whitespace() -> None: + blob = ( + " P with spaces " + "\n\tG with newlines\n" + "\n B\n" + ) + info = parse_synthetic_user_info(blob) + assert info.persona == "P with spaces" + assert info.goal == "G with newlines" + assert info.behavior_guidance == "B" + + +def test_parse_dotall_allows_multiline_content() -> None: + blob = "line 1\nline 2\nline 3G" + info = parse_synthetic_user_info(blob) + assert info.persona == "line 1\nline 2\nline 3" + + +def test_parse_ignores_unknown_tags() -> None: + # Forward-compat: a future generator may add new tags like . + blob = ( + "P" + "G" + "cheerful" + "expert" + ) + info = parse_synthetic_user_info(blob) + assert info.persona == "P" + assert info.goal == "G" + # Unknown tags don't end up anywhere. + assert not hasattr(info, "tone") + + +def test_parse_ignores_preamble_and_trailing() -> None: + blob = "preamble text PG trailing text" + info = parse_synthetic_user_info(blob) + assert info.persona == "P" + assert info.goal == "G" + + +def test_parse_first_match_wins_for_duplicate_tag() -> None: + blob = "firstsecondG" + info = parse_synthetic_user_info(blob) + assert info.persona == "first" + + +def test_parse_missing_persona_raises() -> None: + with pytest.raises(SyntheticUserInfoParseError, match=""): + parse_synthetic_user_info("G") + + +def test_parse_missing_goal_raises() -> None: + with pytest.raises(SyntheticUserInfoParseError, match=""): + parse_synthetic_user_info("P") + + +def test_parse_empty_persona_raises() -> None: + with pytest.raises(SyntheticUserInfoParseError, match=""): + parse_synthetic_user_info("G") + + +def test_parse_whitespace_only_persona_raises() -> None: + # After trim, content is empty — should be treated as missing. + with pytest.raises(SyntheticUserInfoParseError, match=""): + parse_synthetic_user_info(" \n G") + + +def test_parse_completely_unstructured_blob_raises() -> None: + with pytest.raises(SyntheticUserInfoParseError): + parse_synthetic_user_info("just plain text with no tags at all") + + +# ───────────────────────── build ───────────────────────── + + +def test_build_all_three_tags() -> None: + info = SyntheticUserInfo(persona="P", goal="G", behavior_guidance="B") + assert ( + build_synthetic_user_info(info) + == "PGB" + ) + + +def test_build_omits_behavior_guidance_when_none() -> None: + info = SyntheticUserInfo(persona="P", goal="G") + assert build_synthetic_user_info(info) == "PG" + + +def test_build_omits_empty_behavior_guidance() -> None: + # The model permits None for behavior_guidance; build's truthiness check + # also skips empty strings, which matches the parser's "missing → None". + info = SyntheticUserInfo(persona="P", goal="G", behavior_guidance="") + assert build_synthetic_user_info(info) == "PG" + + +# ───────────────────────── roundtrip ───────────────────────── + + +def test_roundtrip_all_three() -> None: + info = SyntheticUserInfo(persona="P\nmultiline", goal="G", behavior_guidance="B") + assert parse_synthetic_user_info(build_synthetic_user_info(info)) == info + + +def test_roundtrip_required_only() -> None: + info = SyntheticUserInfo(persona="P", goal="G") + assert parse_synthetic_user_info(build_synthetic_user_info(info)) == info + + +def test_roundtrip_preserves_internal_whitespace() -> None: + # Whitespace WITHIN content (after the leading/trailing trim) is preserved. + info = SyntheticUserInfo( + persona="word1 word2 word3", goal="G", behavior_guidance=None + ) + assert ( + parse_synthetic_user_info(build_synthetic_user_info(info)).persona + == info.persona + ) diff --git a/libs/core/kiln_ai/synthetic_user/test_prompt.py b/libs/core/kiln_ai/synthetic_user/test_prompt.py new file mode 100644 index 0000000000..f6e21dd00e --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_prompt.py @@ -0,0 +1,95 @@ +"""Unit tests for the persona-playing system prompt template. + +These are structural assertions — we don't check exact wording (that's +allowed to drift as we tune the persona prompt) but we DO check the +prompt has every required section and is free of the removed termination +guidance. +""" + +from kiln_ai.synthetic_user.models import SyntheticUserInfo +from kiln_ai.synthetic_user.prompt import render_system_prompt + + +def _full_info() -> SyntheticUserInfo: + return SyntheticUserInfo( + persona="A 30-something professional, anxious about taxes.", + goal="Find the 2016 RRSP contribution limit.", + behavior_guidance="Press for specific numbers if the agent gives vague answers.", + ) + + +def test_includes_persona_text() -> None: + info = _full_info() + rendered = render_system_prompt(info) + assert info.persona in rendered + + +def test_includes_goal_text() -> None: + info = _full_info() + rendered = render_system_prompt(info) + assert info.goal in rendered + + +def test_includes_behavior_guidance_when_set() -> None: + info = _full_info() + rendered = render_system_prompt(info) + assert info.behavior_guidance is not None + assert info.behavior_guidance in rendered + assert "How you behave" in rendered + + +def test_omits_behavior_guidance_section_when_none() -> None: + info = SyntheticUserInfo(persona="P", goal="G") # behavior_guidance = None + rendered = render_system_prompt(info) + assert "How you behave" not in rendered + + +def test_includes_conventions_block() -> None: + rendered = render_system_prompt(_full_info()) + assert "Conversation style" in rendered + + +def test_does_not_include_termination_sentinels() -> None: + # Decision: drive loop runs for fixed `turns`; SU must NOT try to end + # the conversation. The template should carry no / + # guidance. + rendered = render_system_prompt(_full_info()) + assert "" not in rendered + assert "" not in rendered + assert "DONE" not in rendered # also no bare-word leak + assert "CANCEL" not in rendered + + +def test_section_order_opening_persona_goal_behavior_conventions() -> None: + info = _full_info() + rendered = render_system_prompt(info) + assert info.behavior_guidance is not None + + # Use distinctive substrings from each section. + opening_marker = "Stay in character" + persona_section = "Your persona" + goal_section = "Your goal in this conversation" + behavior_section = "How you behave" + conventions_marker = "Conversation style" + + indexes = [ + rendered.index(opening_marker), + rendered.index(persona_section), + rendered.index(goal_section), + rendered.index(behavior_section), + rendered.index(conventions_marker), + ] + assert indexes == sorted(indexes), f"Sections out of expected order: {indexes}" + + +def test_section_order_skips_behavior_when_none() -> None: + info = SyntheticUserInfo(persona="P", goal="G") + rendered = render_system_prompt(info) + + indexes = [ + rendered.index("Stay in character"), + rendered.index("Your persona"), + rendered.index("Your goal in this conversation"), + rendered.index("Conversation style"), + ] + assert indexes == sorted(indexes) diff --git a/libs/core/kiln_ai/synthetic_user/test_role_swap.py b/libs/core/kiln_ai/synthetic_user/test_role_swap.py new file mode 100644 index 0000000000..4681d900a2 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_role_swap.py @@ -0,0 +1,77 @@ +"""Unit tests for the eval-frame ↔ LLM-frame role swap.""" + +import pytest + +from kiln_ai.synthetic_user.role_swap import role_swap +from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam + + +def test_empty_conversation() -> None: + assert role_swap([]) == [] + + +def test_single_user_becomes_assistant() -> None: + src: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hi"}] + assert role_swap(src) == [{"role": "assistant", "content": "hi"}] + + +def test_single_assistant_becomes_user() -> None: + src: list[ChatCompletionMessageParam] = [{"role": "assistant", "content": "hello"}] + assert role_swap(src) == [{"role": "user", "content": "hello"}] + + +def test_alternating_conversation_flips_both_directions() -> None: + src: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + {"role": "assistant", "content": "a2"}, + ] + expected: list[ChatCompletionMessageParam] = [ + {"role": "assistant", "content": "u1"}, + {"role": "user", "content": "a1"}, + {"role": "assistant", "content": "u2"}, + {"role": "user", "content": "a2"}, + ] + assert role_swap(src) == expected + + +def test_order_preserved() -> None: + src: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": f"m{i}"} for i in range(5) + ] + out = role_swap(src) + assert [m["content"] for m in out] == [f"m{i}" for i in range(5)] + + +def test_multiline_content_passes_through() -> None: + src: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "line 1\nline 2\nline 3"} + ] + assert role_swap(src) == [ + {"role": "assistant", "content": "line 1\nline 2\nline 3"} + ] + + +def test_system_role_raises() -> None: + src: list[ChatCompletionMessageParam] = [{"role": "system", "content": "sys"}] + with pytest.raises(ValueError, match="system"): + role_swap(src) + + +def test_tool_role_raises() -> None: + src: list[ChatCompletionMessageParam] = [ + {"role": "tool", "content": "{}", "tool_call_id": "x"} + ] + with pytest.raises(ValueError, match="tool"): + role_swap(src) + + +def test_does_not_mutate_input() -> None: + src: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + snapshot = [dict(m) for m in src] + role_swap(src) + assert [dict(m) for m in src] == snapshot From b3337daa819578df673cc344196705321acc323b Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:38:38 -0700 Subject: [PATCH 03/36] chore: re-vendor SDK from kiln_server@acbd0ce Picks up an OpenAPI description on GenerateSyntheticUsersResponse.cases documenting the strict-N batch contract. No shape change. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../models/generate_synthetic_users_response.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py index 4ef08c2b99..2c0887017b 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/models/generate_synthetic_users_response.py @@ -24,7 +24,8 @@ class GenerateSyntheticUsersResponse: Clients should re-call to retry. Attributes: - cases (list[SyntheticUserCase]): + cases (list[SyntheticUserCase]): Generated synthetic-user cases. On success, contains exactly `num_cases` items + (the strict-N batch contract — see class docstring). """ cases: list[SyntheticUserCase] From bb002b962f87873faa33496fe74ae3769a1cc30c Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:45:35 -0700 Subject: [PATCH 04/36] feat(studio_server): SyntheticUserClient for /generate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thin async wrapper around the SDK's /v1/synthetic_user/generate endpoint. The SDK now parses 401/422/500/502 into typed response models, so the wrapper switches on the parsed type rather than reading raw bytes — 502 surfaces its typed `code` literal (llm_unavailable | upstream_invalid_output) directly to callers. No retry loop. /generate is a once-per-batch authoring call; kiln_server's pipeline already retries transient provider failures internally before returning 502, so a 502 reaching us is a genuine per-batch failure that should propagate. Drops the v1 client's SyntheticUserTransientError + backoff machinery. No /respond. Per-turn synthetic-user invocation lives at libs/core/kiln_ai/synthetic_user/ and runs locally with the user's keys. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../studio_server/synthetic_user/__init__.py | 22 ++ .../studio_server/synthetic_user/client.py | 187 ++++++++++ .../synthetic_user/test_client.py | 352 ++++++++++++++++++ 3 files changed, 561 insertions(+) create mode 100644 app/desktop/studio_server/synthetic_user/__init__.py create mode 100644 app/desktop/studio_server/synthetic_user/client.py create mode 100644 app/desktop/studio_server/synthetic_user/test_client.py diff --git a/app/desktop/studio_server/synthetic_user/__init__.py b/app/desktop/studio_server/synthetic_user/__init__.py new file mode 100644 index 0000000000..72b84653f6 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/__init__.py @@ -0,0 +1,22 @@ +"""Studio_server-side wrapper for the kiln_server synthetic-user `/generate` +endpoint, plus the multi-turn synthetic-data-generation drive loop. + +Per-turn synthetic-user invocation itself lives in +`libs/core/kiln_ai/synthetic_user/` (calls the LLM with the user's keys); +this module covers only the authoring HTTP call and the runner that +orchestrates target invocation + SU response across multiple cases. +""" + +from app.desktop.studio_server.synthetic_user.client import ( + SyntheticUserClient, + SyntheticUserError, + SyntheticUserRequestError, + SyntheticUserServerError, +) + +__all__ = [ + "SyntheticUserClient", + "SyntheticUserError", + "SyntheticUserRequestError", + "SyntheticUserServerError", +] diff --git a/app/desktop/studio_server/synthetic_user/client.py b/app/desktop/studio_server/synthetic_user/client.py new file mode 100644 index 0000000000..c42c88187d --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/client.py @@ -0,0 +1,187 @@ +"""Thin async wrapper over the vendored kiln_server SDK for `/generate`. + +Owns two concerns the SDK leaves to callers: + +1. Error classification. The SDK parses 200/401/422/500/502 into typed + models; we translate those into the wrapper's typed exception hierarchy + so callers never inspect raw HTTP status codes. +2. No retry. `/generate` is a once-per-batch authoring call (not a per-turn + hot loop), and kiln_server's own pipeline already retries transient + provider failures once before returning 502. A 502 reaching us is a + genuine per-batch failure that should surface, not a transient to retry. + +No `/respond` here. Per-turn synthetic-user invocation lives in +`libs/core/kiln_ai/synthetic_user/`. This wrapper exists only for the +authoring call. +""" + +import logging + +from app.desktop.studio_server.api_client.kiln_ai_server_client.api.synthetic_user import ( + generate_v1_synthetic_user_generate_post, +) +from app.desktop.studio_server.api_client.kiln_ai_server_client.client import ( + AuthenticatedClient, +) +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + GenerateSyntheticUsersRequest, + GenerateSyntheticUsersResponse, + GenerateV1SyntheticUserGeneratePostResponse401, + GenerateV1SyntheticUserGeneratePostResponse500, + GenerateV1SyntheticUserGeneratePostResponse502, + HTTPValidationError, + SyntheticUserCase, + ValidationError, +) +from app.desktop.studio_server.api_client.kiln_ai_server_client.types import ( + UNSET, + Response, +) +from app.desktop.studio_server.api_client.kiln_server_client import ( + get_authenticated_client, +) + +logger = logging.getLogger(__name__) + + +class SyntheticUserError(Exception): + """Base class for SyntheticUserClient errors. Carries the kiln_server + error code (e.g. `llm_unavailable`, `upstream_invalid_output`) when + available, plus the HTTP status for debugging. + """ + + def __init__(self, code: str, message: str, status_code: int | None = None): + prefix = f"{code}: " if code else "" + super().__init__(f"{prefix}{message}") + self.code = code + self.message = message + self.status_code = status_code + + +class SyntheticUserRequestError(SyntheticUserError): + """Raised on 4xx. Either we sent a bad body (422 — runner bug) or the + caller's credentials don't work (401). Not retryable — fix inputs. + """ + + +class SyntheticUserServerError(SyntheticUserError): + """Raised on 5xx. 500 is a server bug; 502 is the kiln_server pipeline + giving up on the upstream provider after its own internal retry. Not + retryable at this layer — bubble up as a per-batch failure. + """ + + +class SyntheticUserClient: + """Async client for kiln_server's `/v1/synthetic_user/generate`. + + Construction is cheap — no network until a method is called. Pass the + same instance to as many coroutines as you want; the underlying httpx + client is async-safe. + """ + + def __init__(self, *, api_key: str): + self._client: AuthenticatedClient = get_authenticated_client(api_key=api_key) + + async def generate( + self, + *, + target_task_prompt: str, + target_specification: str, + num_cases: int, + ) -> list[SyntheticUserCase]: + """POST /v1/synthetic_user/generate. Returns the SDK's case models + as-is; each carries `seed_prompt` and `synthetic_user_info` (the + tagged blob — see kiln_ai.synthetic_user.parser for the schema). + """ + body = GenerateSyntheticUsersRequest( + target_task_prompt=target_task_prompt, + target_specification=target_specification, + num_cases=num_cases, + ) + response = await generate_v1_synthetic_user_generate_post.asyncio_detailed( + client=self._client, body=body + ) + return self._extract_cases_or_raise(response) + + @staticmethod + def _extract_cases_or_raise(response: Response) -> list[SyntheticUserCase]: + """Translate the SDK's parsed response into either the case list + (on 2xx) or a typed exception (on anything else). + """ + parsed = response.parsed + status = int(response.status_code) + + if isinstance(parsed, GenerateSyntheticUsersResponse): + return list(parsed.cases) + + # Typed error bodies the SDK parses for us. + if isinstance(parsed, GenerateV1SyntheticUserGeneratePostResponse502): + # `code` is a typed enum on 502; surface its string value so + # downstream callers can discriminate llm_unavailable from + # upstream_invalid_output without importing the SDK type. + raise SyntheticUserServerError( + code=parsed.code.value, + message=parsed.message, + status_code=status, + ) + if isinstance(parsed, GenerateV1SyntheticUserGeneratePostResponse500): + raise SyntheticUserServerError( + code=_code_or_default(parsed.code, f"http_{status}"), + message=parsed.message, + status_code=status, + ) + if isinstance(parsed, GenerateV1SyntheticUserGeneratePostResponse401): + raise SyntheticUserRequestError( + code=_code_or_default(parsed.code, "unauthorized"), + message=parsed.message, + status_code=status, + ) + if isinstance(parsed, HTTPValidationError): + # 422 — we sent a body the server's pydantic validator rejected. + # Indicates a runner bug; surface enough detail to debug. + raise SyntheticUserRequestError( + code="http_422", + message=_format_validation_detail(parsed), + status_code=status, + ) + + # Fallback: SDK couldn't parse a body, or the response is otherwise + # unexpected. Pick a coarse classification by status range. + if 400 <= status < 500: + raise SyntheticUserRequestError( + code=f"http_{status}", + message="Unexpected client-error response from kiln_server.", + status_code=status, + ) + raise SyntheticUserServerError( + code=f"http_{status}", + message="Unexpected response from kiln_server.", + status_code=status, + ) + + +def _code_or_default(code: object, default: str) -> str: + """Resolve a typed-model `code` field that may be `UNSET` or a string.""" + if isinstance(code, str) and code: + return code + if code is UNSET or code is None: + return default + return default + + +def _format_validation_detail(error: HTTPValidationError) -> str: + """Render a FastAPI HTTPValidationError into a single-line message + useful for debugging which field the runner sent wrong. + """ + detail = error.detail + if not isinstance(detail, list): + return "Validation error (no detail)." + parts: list[str] = [] + for item in detail: + if not isinstance(item, ValidationError): + continue + loc = ".".join(str(x) for x in item.loc) + parts.append(f"{loc}: {item.msg}") + if not parts: + return "Validation error (no detail)." + return "Validation error: " + "; ".join(parts) diff --git a/app/desktop/studio_server/synthetic_user/test_client.py b/app/desktop/studio_server/synthetic_user/test_client.py new file mode 100644 index 0000000000..0d8637ad54 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/test_client.py @@ -0,0 +1,352 @@ +"""Unit tests for SyntheticUserClient (the `/generate` wrapper). + +The SDK's `asyncio_detailed` is patched per-test so no real network call +happens. Tests cover each status the SDK models (200, 401, 422, 500, 502) ++ the fallback paths for unparseable bodies. +""" + +from http import HTTPStatus +from unittest.mock import AsyncMock + +import pytest + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + GenerateSyntheticUsersResponse, + GenerateV1SyntheticUserGeneratePostResponse401, + GenerateV1SyntheticUserGeneratePostResponse500, + GenerateV1SyntheticUserGeneratePostResponse502, + GenerateV1SyntheticUserGeneratePostResponse502Code, + HTTPValidationError, + SyntheticUserCase, + ValidationError, +) +from app.desktop.studio_server.api_client.kiln_ai_server_client.types import ( + UNSET, + Response, +) +from app.desktop.studio_server.synthetic_user import client as client_mod +from app.desktop.studio_server.synthetic_user.client import ( + SyntheticUserClient, + SyntheticUserRequestError, + SyntheticUserServerError, +) + + +def _make_client() -> SyntheticUserClient: + return SyntheticUserClient(api_key="test-key") + + +def _patch_generate(monkeypatch: pytest.MonkeyPatch, mock: AsyncMock) -> None: + monkeypatch.setattr( + client_mod.generate_v1_synthetic_user_generate_post, + "asyncio_detailed", + mock, + ) + + +def _ok_response(num_cases: int = 1) -> Response: + cases = [ + SyntheticUserCase( + seed_prompt=f"seed-{i}", + synthetic_user_info=( + f"persona-{i}" + f"goal-{i}" + f"guidance-{i}" + ), + ) + for i in range(num_cases) + ] + return Response( + status_code=HTTPStatus.OK, + content=b"{}", + headers={}, + parsed=GenerateSyntheticUsersResponse(cases=cases), + ) + + +def _err_response(status: int, parsed: object) -> Response: + return Response( + status_code=HTTPStatus(status), + content=b"{}", + headers={}, + parsed=parsed, # type: ignore[arg-type] + ) + + +# ───────────────────────── happy path ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_happy_path_returns_cases( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_generate(monkeypatch, AsyncMock(return_value=_ok_response(num_cases=3))) + + cases = await _make_client().generate( + target_task_prompt="prompt", + target_specification="spec", + num_cases=3, + ) + + assert len(cases) == 3 + assert cases[0].seed_prompt == "seed-0" + # synthetic_user_info is the tagged blob — opaque at this layer. + assert "persona-0" in cases[0].synthetic_user_info + + +@pytest.mark.asyncio +async def test_generate_passes_request_body_correctly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list = [] + + async def _capture(*, client, body): # noqa: ARG001 + captured.append(body) + return _ok_response() + + _patch_generate(monkeypatch, AsyncMock(side_effect=_capture)) + + await _make_client().generate( + target_task_prompt="my task prompt", + target_specification="my spec", + num_cases=5, + ) + + assert len(captured) == 1 + sent = captured[0] + assert sent.target_task_prompt == "my task prompt" + assert sent.target_specification == "my spec" + assert sent.num_cases == 5 + + +# ───────────────────────── 502 (typed code) ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_502_llm_unavailable_surfaces_typed_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse502( + message="provider timed out", + code=GenerateV1SyntheticUserGeneratePostResponse502Code.LLM_UNAVAILABLE, + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(502, parsed))) + + with pytest.raises(SyntheticUserServerError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "llm_unavailable" + assert exc.value.message == "provider timed out" + assert exc.value.status_code == 502 + + +@pytest.mark.asyncio +async def test_generate_502_upstream_invalid_output_surfaces_typed_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse502( + message="model returned unparseable output", + code=GenerateV1SyntheticUserGeneratePostResponse502Code.UPSTREAM_INVALID_OUTPUT, + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(502, parsed))) + + with pytest.raises(SyntheticUserServerError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "upstream_invalid_output" + + +# ───────────────────────── 500 ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_500_with_code(monkeypatch: pytest.MonkeyPatch) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse500( + message="kaboom", + code="internal_error", + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(500, parsed))) + + with pytest.raises(SyntheticUserServerError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "internal_error" + assert exc.value.message == "kaboom" + assert exc.value.status_code == 500 + + +@pytest.mark.asyncio +async def test_generate_500_with_unset_code_falls_back( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse500( + message="kaboom", + code=UNSET, # type: ignore[arg-type] + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(500, parsed))) + + with pytest.raises(SyntheticUserServerError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "http_500" + + +# ───────────────────────── 401 ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_401_surfaces_as_request_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse401( + message="invalid api key", + code="unauthorized", + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(401, parsed))) + + with pytest.raises(SyntheticUserRequestError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "unauthorized" + assert exc.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_generate_401_with_unset_code_uses_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = GenerateV1SyntheticUserGeneratePostResponse401( + message="invalid api key", + code=UNSET, # type: ignore[arg-type] + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(401, parsed))) + + with pytest.raises(SyntheticUserRequestError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "unauthorized" + + +# ───────────────────────── 422 (HTTPValidationError) ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_422_renders_validation_detail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + parsed = HTTPValidationError( + detail=[ + ValidationError( + loc=["body", "num_cases"], + msg="value is greater than 50", + type_="value_error", + ), + ValidationError( + loc=["body", "target_specification"], + msg="field required", + type_="missing", + ), + ] + ) + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(422, parsed))) + + with pytest.raises(SyntheticUserRequestError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=999 + ) + + # Code is the http_422 sentinel; message carries the structured detail. + assert exc.value.code == "http_422" + assert "num_cases" in exc.value.message + assert "value is greater than 50" in exc.value.message + assert "target_specification" in exc.value.message + + +@pytest.mark.asyncio +async def test_generate_422_with_no_detail_returns_generic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # SDK's HTTPValidationError accepts an Unset detail; we render a + # generic message rather than crashing. + parsed = HTTPValidationError(detail=UNSET) # type: ignore[arg-type] + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(422, parsed))) + + with pytest.raises(SyntheticUserRequestError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "http_422" + assert "no detail" in exc.value.message.lower() + + +# ───────────────────────── unparseable / unexpected ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_unparseable_4xx_falls_back_to_request_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """parsed=None on a 4xx (SDK didn't recognize the body) — generic mapping.""" + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(400, None))) + + with pytest.raises(SyntheticUserRequestError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "http_400" + assert exc.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_generate_unparseable_5xx_falls_back_to_server_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_generate(monkeypatch, AsyncMock(return_value=_err_response(503, None))) + + with pytest.raises(SyntheticUserServerError) as exc: + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + assert exc.value.code == "http_503" + assert exc.value.status_code == 503 + + +# ───────────────────────── no retry surface ───────────────────────── + + +@pytest.mark.asyncio +async def test_generate_does_not_retry_on_502( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Unlike the v1 client, /generate has no retry loop — a 502 surfaces + immediately as a per-batch failure. + """ + parsed = GenerateV1SyntheticUserGeneratePostResponse502( + message="boom", + code=GenerateV1SyntheticUserGeneratePostResponse502Code.LLM_UNAVAILABLE, + ) + mock = AsyncMock(return_value=_err_response(502, parsed)) + _patch_generate(monkeypatch, mock) + + with pytest.raises(SyntheticUserServerError): + await _make_client().generate( + target_task_prompt="p", target_specification="s", num_cases=1 + ) + + # Exactly one call — no retry budget consumed. + assert mock.await_count == 1 From 322d815e739035df096679fbb25b6d576fa15329 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:52:56 -0700 Subject: [PATCH 05/36] feat(libs/core/synthetic_user): tighten persona prompt opening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an explicit "your entire output is the user's next message, verbatim and nothing else: no narration, no meta-commentary, no quotes, no labels like 'User:'" clause to the persona-playing system prompt. A team running similar SU-driven evals reported the persona-playing model frequently breaks character — narrating ("I would now ask..."), self-evaluating, or labeling its output. This clause pins that down at the prompt boundary so we don't end up reaching for post-processing band-aids later. Co-Authored-By: Claude Opus 4.7 (1M context) --- libs/core/kiln_ai/synthetic_user/prompt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/core/kiln_ai/synthetic_user/prompt.py b/libs/core/kiln_ai/synthetic_user/prompt.py index f9d0521d56..2576cf4b3b 100644 --- a/libs/core/kiln_ai/synthetic_user/prompt.py +++ b/libs/core/kiln_ai/synthetic_user/prompt.py @@ -10,7 +10,10 @@ _OPENING = ( "You are playing the role of a user interacting with an AI assistant. " - "Stay in character — respond as the user would, not as the assistant." + "Stay in character — respond as the user would, not as the assistant. " + "Your entire output is the user's next message, verbatim and nothing " + "else: no narration, no meta-commentary, no quotes, no labels like " + '"User:". Just the message text the user would type.' ) _CONVENTIONS = """## Conversation style From ab74931a3c8a4a6ca42b7597aa3c648a1e52874e Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:26:46 -0700 Subject: [PATCH 06/36] feat(studio_server): drive_case + run_cases_batch with local SU driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drive_loop.py: - drive_case(*, case, target_invoker, su_driver, turns, on_turn) runs the loop for exactly `turns` iterations — no early termination, no stop_signal plumbing. Returns DriveCaseResult(chain) with the persisted TaskRun chain. - TargetInvoker + TurnHook Protocols. The SU driver does all role filtering / role swap / invariant checks internally; the drive loop passes the cumulative trace as-is. runner.py: - run_cases_batch is an async generator yielding typed BatchEvents (BatchStartedEvent / TurnCompletedEvent / CaseCompletedEvent / CaseFailedEvent / BatchCompletedEvent). No stop_signal/stop_reason fields — drive loop is fixed-length. - Constructs a SyntheticUserDriver per case; a malformed synthetic_user_info blob surfaces as a CaseFailedEvent for that case alone (other cases continue). - _make_target_invoker / _build_input_source / _tag_leaf patterns kept from the prior v1 commits (target persistence + SU attribution unchanged). input_source now carries the opaque blob on the root run + slim {batch_tag, turn_index} on subsequent turns. - Per-case try/except now WRAPS _tag_leaf too, so a save_to_file failure surfaces as case_failed instead of silently disappearing into asyncio.gather(return_exceptions=True). Same try also wraps the target_invoker construction. - Case tasks are kicked off before the first BatchStartedEvent yield and the entire drain loop is inside a try/finally that cancels them on consumer disconnect — fixes the v1 issue where browser disconnect kept the request alive for the full duration of every in-flight case. 14 tests cover: input validation, happy-path event stream, leaf tagging, auto-generated batch_tag, malformed blob → case_failed, target invoke failure → case_failed, tag-save failure → case_failed, concurrency semaphore enforcing max-in-flight, root vs slim input_source attribution, and consumer cancellation propagating to case tasks. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../synthetic_user/drive_loop.py | 119 ++++ .../studio_server/synthetic_user/runner.py | 431 +++++++++++++ .../synthetic_user/test_drive_loop.py | 301 +++++++++ .../synthetic_user/test_runner.py | 576 ++++++++++++++++++ 4 files changed, 1427 insertions(+) create mode 100644 app/desktop/studio_server/synthetic_user/drive_loop.py create mode 100644 app/desktop/studio_server/synthetic_user/runner.py create mode 100644 app/desktop/studio_server/synthetic_user/test_drive_loop.py create mode 100644 app/desktop/studio_server/synthetic_user/test_runner.py diff --git a/app/desktop/studio_server/synthetic_user/drive_loop.py b/app/desktop/studio_server/synthetic_user/drive_loop.py new file mode 100644 index 0000000000..93dea6cd11 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/drive_loop.py @@ -0,0 +1,119 @@ +"""Single-case drive loop for the multi-turn synthetic-user runner. + +`drive_case` alternates the target task adapter with the local +`SyntheticUserDriver`, producing a chain of persisted `TaskRun`s on the +target side. The loop runs for a fixed `turns` count — no early +termination, no `` / `` sentinels — by design (see spec). + +Persistence is fully delegated to the adapter: every TaskRun in the +returned chain has already been written to disk by `target_invoker(...)`, +with `trace`, `parent_task_run_id`, and `cumulative_usage` populated. +The SU side is in-memory only and produces no TaskRuns. +""" + +from dataclasses import dataclass +from typing import Protocol + +from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.driver import SyntheticUserDriver +from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) + + +class TargetInvoker(Protocol): + """Callable that invokes the target task for one turn. Phase 3 wraps + `adapter_for_task(task, run_config).invoke` to satisfy this; tests + pass in a fake. Keeps the drive loop target-agnostic — it just cares + about the persisted TaskRun that comes back. + """ + + async def __call__( + self, + *, + input: str, + prior_trace: list[ChatCompletionMessageParam] | None, + parent_task_run: TaskRun | None, + ) -> TaskRun: ... + + +class TurnHook(Protocol): + """Optional callback invoked once per turn after the SU has replied. + + The runner uses this to translate per-turn outcomes into BatchEvents + without coupling drive_case to the event shape. The hook fires after + both the assistant turn is persisted AND the SU's next message is + produced, so callers have all per-turn signal in one place. + """ + + async def __call__(self, *, run: TaskRun, su_message: str) -> None: ... + + +@dataclass(frozen=True) +class DriveCaseResult: + """Outcome of one drive_case run. + + `chain` is the list of persisted TaskRuns the adapter produced (leaf + last). There is no stop_reason field — every case ends after exactly + `turns` iterations by design. + """ + + chain: list[TaskRun] + + +async def drive_case( + *, + case: SyntheticUserCase, + target_invoker: TargetInvoker, + su_driver: SyntheticUserDriver, + turns: int, + on_turn: TurnHook | None = None, +) -> DriveCaseResult: + """Drive one synthetic-user case for `turns` turns. + + Args: + case: the generated SU case (carries seed_prompt + the + opaque synthetic_user_info blob the driver was built from). + target_invoker: how to call the target task; produces a persisted TaskRun. + su_driver: pre-built SU driver for this case. Caller is responsible + for construction (so a malformed blob fails at the runner layer, + not here). + turns: exact number of assistant turns to produce. The loop runs + `range(turns)` and always completes all iterations — no early + stop. + on_turn: optional async hook called once per turn after `su_driver.respond` + returns. The runner plugs in here to emit TurnCompletedEvent. + + Returns: + DriveCaseResult with the chain of TaskRuns produced (leaf last). + """ + if turns < 1: + raise ValueError(f"turns must be >= 1, got {turns}") + + user_msg: str = case.seed_prompt + prev_run: TaskRun | None = None + prev_trace: list[ChatCompletionMessageParam] | None = None + chain: list[TaskRun] = [] + + for _ in range(turns): + new_run = await target_invoker( + input=user_msg, + prior_trace=prev_trace, + parent_task_run=prev_run, + ) + chain.append(new_run) + + # The SU driver does the role filtering / role swap / invariant + # checks itself. We pass the new run's cumulative trace as-is. + su_message = await su_driver.respond(new_run.trace or []) + + if on_turn is not None: + await on_turn(run=new_run, su_message=su_message) + + user_msg = su_message + prev_run = new_run + prev_trace = new_run.trace + + return DriveCaseResult(chain=chain) diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py new file mode 100644 index 0000000000..fc5a31b414 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -0,0 +1,431 @@ +"""Batch runner — fans drive_case out across N cases, streams BatchEvents. + +`run_cases_batch` is an async generator yielding typed events so the +upcoming SSE endpoint can map each to a `data:` frame without +re-instrumenting. Cases run concurrently under an asyncio.Semaphore. + +Responsibilities owned here (not in drive_case): +- Building the per-case `SyntheticUserDriver` (catches malformed + `synthetic_user_info` blob → `CaseFailedEvent` for that case only). +- Building the per-case `TargetInvoker` that wraps `adapter.invoke` with + the SU-attribution `input_source` — opaque blob on the root run, slim + `{batch_tag, turn_index}` on subsequent turns. +- Tagging the leaf TaskRun for downstream eval-dataset discovery. +- Per-case error isolation (typed driver / loop / unexpected exceptions + → `CaseFailedEvent` without affecting other in-flight cases). +- Bounded cleanup on consumer disconnect — case tasks are cancelled + before the closer awaits them, so a browser disconnect doesn't keep + the request alive for the duration of every in-flight case. +""" + +import asyncio +import contextlib +import logging +import uuid +from dataclasses import dataclass +from typing import AsyncIterator + +from kiln_ai.adapters.adapter_registry import adapter_for_task +from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties +from kiln_ai.datamodel.task import Task +from kiln_ai.datamodel.task_output import DataSource, DataSourceType +from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.driver import SyntheticUserDriver +from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig +from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError +from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) +from app.desktop.studio_server.synthetic_user.drive_loop import ( + TargetInvoker, + drive_case, +) + +logger = logging.getLogger(__name__) + +# Module constants. Sized for an MVP beta — easy to bump in one place. +NUM_CASES_MAX = 10 +DEFAULT_TURNS = 5 +CONCURRENCY = 4 + +# Tag scheme. `_TAG_SU_CASE` lets the dataset view filter to all SU-generated +# leaves; `_TAG_PREFIX_SU_BATCH` (+ batch_tag) groups one batch for review. +_TAG_SU_CASE = "synthetic_user_case" +_TAG_PREFIX_SU_BATCH = "synthetic_user_batch:" + +# Identifies this code path in `input_source.properties.adapter_name` so a +# reader looking at a TaskRun can tell who created it. +_RUNNER_ADAPTER_NAME = "kiln_synthetic_user_runner" + + +# ───────────────────────── BatchEvent dataclasses ───────────────────────── + + +@dataclass(frozen=True) +class BatchStartedEvent: + batch_tag: str + num_cases: int + + +@dataclass(frozen=True) +class TurnCompletedEvent: + case_index: int + assistant_run_id: str + su_next_message: str + cumulative_cost: float + # Cumulative OpenAI-format trace at this point (system + all turns so far). + # Lets the UI render the live conversation without a follow-up fetch. + trace: list[ChatCompletionMessageParam] + + +@dataclass(frozen=True) +class CaseCompletedEvent: + case_index: int + chain_run_ids: list[str] + leaf_run_id: str + total_turns: int + total_cost: float + + +@dataclass(frozen=True) +class CaseFailedEvent: + case_index: int + error_code: str + message: str + + +@dataclass(frozen=True) +class BatchCompletedEvent: + successful: int + failed: int + batch_tag: str + total_cost: float + + +BatchEvent = ( + BatchStartedEvent + | TurnCompletedEvent + | CaseCompletedEvent + | CaseFailedEvent + | BatchCompletedEvent +) + + +# ───────────────────────── public entry point ───────────────────────── + + +async def run_cases_batch( + *, + cases: list[SyntheticUserCase], + target_task: Task, + target_run_config: KilnAgentRunConfigProperties, + su_driver_config: SyntheticUserDriverConfig, + turns: int = DEFAULT_TURNS, + concurrency: int = CONCURRENCY, + batch_tag: str | None = None, +) -> AsyncIterator[BatchEvent]: + """Drive `cases` concurrently against `target_task`, streaming progress. + + Each case runs in its own coroutine under an `asyncio.Semaphore` so the + target-task LLM and the SU LLM aren't both hammered at unbounded fan-out. + Events from different cases interleave; ordering WITHIN a case is + `turn_completed`* → `case_completed | case_failed`. + + Yields: + BatchStartedEvent — once, before any case runs. + TurnCompletedEvent — one per assistant turn within a case. + CaseCompletedEvent — one per case that ran to completion. + CaseFailedEvent — one per case that hit a typed error. + BatchCompletedEvent — once, after all cases finish. + """ + if not cases: + raise ValueError("cases cannot be empty") + if turns < 1: + raise ValueError("turns must be >= 1") + if concurrency < 1: + raise ValueError("concurrency must be >= 1") + + resolved_batch_tag = batch_tag or _new_batch_tag() + semaphore = asyncio.Semaphore(concurrency) + # `None` is the end-of-stream sentinel pushed when all cases finish. + queue: asyncio.Queue[BatchEvent | None] = asyncio.Queue() + + async def _drive_one(case_index: int, case: SyntheticUserCase) -> None: + async with semaphore: + await _drive_one_case_and_emit( + case_index=case_index, + case=case, + target_task=target_task, + target_run_config=target_run_config, + su_driver_config=su_driver_config, + turns=turns, + batch_tag=resolved_batch_tag, + queue=queue, + ) + + # Kick the cases off BEFORE the first yield so they start running + # concurrently with consumer setup. If the consumer disconnects between + # BatchStartedEvent and the drain loop, the `finally` below still + # cancels them. + case_tasks = [asyncio.create_task(_drive_one(i, c)) for i, c in enumerate(cases)] + + async def _close_when_done() -> None: + # `return_exceptions=True` so a stray bug doesn't leave the queue + # draining forever; we surface failures via per-case CaseFailedEvent. + await asyncio.gather(*case_tasks, return_exceptions=True) + await queue.put(None) + + closer = asyncio.create_task(_close_when_done()) + + successful = 0 + failed = 0 + total_cost = 0.0 + try: + yield BatchStartedEvent(batch_tag=resolved_batch_tag, num_cases=len(cases)) + + while True: + event = await queue.get() + if event is None: + break + yield event + if isinstance(event, CaseCompletedEvent): + successful += 1 + total_cost += event.total_cost + elif isinstance(event, CaseFailedEvent): + failed += 1 + + yield BatchCompletedEvent( + successful=successful, + failed=failed, + batch_tag=resolved_batch_tag, + total_cost=total_cost, + ) + finally: + # Cancel any in-flight case tasks before awaiting the closer. Without + # this, a consumer disconnect (e.g. browser closes the SSE) leaves + # workers running to completion writing to a dead queue — the request + # stays "alive" for the full duration of every case. + for task in case_tasks: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await closer + + +# ───────────────────────── per-case orchestration ───────────────────────── + + +async def _drive_one_case_and_emit( + *, + case_index: int, + case: SyntheticUserCase, + target_task: Task, + target_run_config: KilnAgentRunConfigProperties, + su_driver_config: SyntheticUserDriverConfig, + turns: int, + batch_tag: str, + queue: asyncio.Queue[BatchEvent | None], +) -> None: + """Run drive_case for one case, translating per-turn outcomes to events + on `queue` and ending with either CaseCompletedEvent or CaseFailedEvent. + + All upstream work — SyntheticUserDriver construction, target_invoker + construction, drive_case execution, leaf tagging — runs inside a single + try/except so any failure surfaces as a CaseFailedEvent rather than + escaping into `asyncio.gather(return_exceptions=True)` (which would + silently drop the case from the event stream). + """ + try: + # Construct the per-case SU driver. Parses the blob immediately; + # a malformed blob fails this case without affecting others. + try: + su_driver = SyntheticUserDriver(case.synthetic_user_info, su_driver_config) + except SyntheticUserInfoParseError as e: + await queue.put( + CaseFailedEvent( + case_index=case_index, + error_code="bad_synthetic_user_info", + message=str(e), + ) + ) + return + + target_invoker = _make_target_invoker( + case=case, + target_task=target_task, + target_run_config=target_run_config, + su_driver_config=su_driver_config, + batch_tag=batch_tag, + ) + + async def _on_turn(*, run: TaskRun, su_message: str) -> None: + await queue.put( + TurnCompletedEvent( + case_index=case_index, + assistant_run_id=str(run.id) if run.id is not None else "", + su_next_message=su_message, + cumulative_cost=_cumulative_cost(run), + # Snapshot the cumulative trace so the UI can render + # the live conversation without a follow-up fetch. + trace=list(run.trace) if run.trace else [], + ) + ) + + result = await drive_case( + case=case, + target_invoker=target_invoker, + su_driver=su_driver, + turns=turns, + on_turn=_on_turn, + ) + + # Tag the leaf so eval-time loaders can find it. Inside the try + # so a tag-save failure (full disk, validator rejection on a + # malformed batch_tag) surfaces as case_failed, not a silent drop. + leaf = result.chain[-1] + _tag_leaf(leaf, batch_tag) + + await queue.put( + CaseCompletedEvent( + case_index=case_index, + chain_run_ids=[ + str(r.id) if r.id is not None else "" for r in result.chain + ], + leaf_run_id=str(leaf.id) if leaf.id is not None else "", + total_turns=len(result.chain), + total_cost=_cumulative_cost(leaf), + ) + ) + except Exception as e: # noqa: BLE001 — beta error surface + # Adapter network errors, model misconfig, save_to_file blow-up, + # anything unexpected. Log with full traceback; emit a structured + # failure so the batch invariant holds (every case gets one event). + logger.exception( + "synthetic_user runner: unexpected error in case %d", case_index + ) + await queue.put( + CaseFailedEvent( + case_index=case_index, + error_code="unexpected_error", + message=f"{type(e).__name__}: {e}", + ) + ) + + +# ───────────────────────── target invoker construction ───────────────────── + + +def _make_target_invoker( + *, + case: SyntheticUserCase, + target_task: Task, + target_run_config: KilnAgentRunConfigProperties, + su_driver_config: SyntheticUserDriverConfig, + batch_tag: str, +) -> TargetInvoker: + """Build a per-case TargetInvoker over the real adapter. + + The closure tracks `turn_index` so the root run carries the full SU + attribution context in `input_source.properties` while subsequent + runs carry only the slim `{batch_tag, turn_index}` — the case is + recoverable by walking `parent_task_run_id` to the root. + """ + adapter = adapter_for_task(target_task, target_run_config) + turn_index = 0 + + async def _invoker( + *, + input: str, + prior_trace: list[ChatCompletionMessageParam] | None, + parent_task_run: TaskRun | None, + ) -> TaskRun: + nonlocal turn_index + turn_index += 1 + input_source = _build_input_source( + case=case, + su_driver_config=su_driver_config, + batch_tag=batch_tag, + turn_index=turn_index, + is_root=(turn_index == 1), + ) + return await adapter.invoke( + input=input, + input_source=input_source, + prior_trace=prior_trace, + parent_task_run=parent_task_run, + ) + + return _invoker + + +def _build_input_source( + *, + case: SyntheticUserCase, + su_driver_config: SyntheticUserDriverConfig, + batch_tag: str, + turn_index: int, + is_root: bool, +) -> DataSource: + """Attribute the user-side input on this turn to the SU driver model. + + Reuses `DataSourceType.synthetic` (a model produced this text) rather + than inventing a new type — the existing validator accepts arbitrary + extra property keys alongside the required `model_name` / + `model_provider` / `adapter_name`. + + Root run carries the full opaque blob + seed_prompt so anyone landing + on this run can recover the case context without walking the chain. + Subsequent runs carry only the slim batch_tag/turn_index pair. + """ + props: dict[str, str | int | float] = { + "model_name": su_driver_config.model_name, + "model_provider": su_driver_config.model_provider_name.value, + "adapter_name": _RUNNER_ADAPTER_NAME, + "batch_tag": batch_tag, + "turn_index": turn_index, + } + if is_root: + props["synthetic_user_info"] = case.synthetic_user_info + props["seed_prompt"] = case.seed_prompt + + # The validator rejects empty string property values. Strip any that + # would trip it (shouldn't happen for required fields in practice, but + # this protects against an empty seed_prompt or a malformed config). + filtered = {k: v for k, v in props.items() if not (isinstance(v, str) and v == "")} + return DataSource(type=DataSourceType.synthetic, properties=filtered) + + +# ───────────────────────── small utilities ───────────────────────── + + +def _new_batch_tag() -> str: + """12-char hex tag from uuid4 — short enough to read; long enough to + avoid collisions across batches a user runs in the same session. + """ + return uuid.uuid4().hex[:12] + + +def _cumulative_cost(run: TaskRun) -> float: + """Read the rolled-up cost from a TaskRun, defaulting to 0 if usage is + missing (defensive against fakes in unit tests that don't populate it). + """ + usage = getattr(run, "cumulative_usage", None) + if usage is None: + return 0.0 + return float(getattr(usage, "cost", None) or 0.0) + + +def _tag_leaf(leaf: TaskRun, batch_tag: str) -> None: + """Add the runner's discovery tags to the leaf TaskRun and persist. + + Tags are deduplicated (treated as a set then sorted) so re-runs + against an already-tagged leaf are idempotent. A save_to_file + exception surfaces to the caller (which converts to CaseFailedEvent). + """ + tags = set(leaf.tags or []) + tags.add(_TAG_SU_CASE) + tags.add(f"{_TAG_PREFIX_SU_BATCH}{batch_tag}") + leaf.tags = sorted(tags) + leaf.save_to_file() diff --git a/app/desktop/studio_server/synthetic_user/test_drive_loop.py b/app/desktop/studio_server/synthetic_user/test_drive_loop.py new file mode 100644 index 0000000000..d4e9f2f420 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/test_drive_loop.py @@ -0,0 +1,301 @@ +"""Unit tests for drive_case. + +The TargetInvoker is replaced with a hand-written fake; the +SyntheticUserDriver is replaced with a `Mock(spec=)` whose `respond()` +is an AsyncMock returning canned strings. No real model calls. +""" + +from typing import Any +from unittest.mock import AsyncMock, Mock + +import pytest + +from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.driver import SyntheticUserDriver + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) +from app.desktop.studio_server.synthetic_user.drive_loop import ( + DriveCaseResult, + drive_case, +) + +# ───────────────────────── helpers / fixtures ───────────────────────── + + +_SYSTEM_PROMPT = "You are a target agent." + + +def _case(seed: str = "hi there") -> SyntheticUserCase: + return SyntheticUserCase( + seed_prompt=seed, + synthetic_user_info=( + "frustrated customer" + "get a refund outside policy" + "be polite then escalate" + ), + ) + + +def _fake_run(trace: list[dict], run_id: str | None = None) -> Mock: + """Minimal stand-in for a persisted TaskRun — drive_case only reads + `.trace`; we add `.id` so tests can identify the chain order. + """ + run = Mock(spec=TaskRun) + run.trace = trace + run.id = run_id or f"run-{len(trace)}" + return run + + +class _FakeInvoker: + """Records each call's (input, prior_trace, parent_task_run) and + returns successive canned TaskRun mocks. Each call appends one + user + one assistant turn to the prior trace to mimic how + `adapter.invoke` builds cumulative traces. + """ + + def __init__(self, assistant_replies: list[str]): + self._replies = list(assistant_replies) + self.calls: list[dict[str, Any]] = [] + + async def __call__( + self, + *, + input: str, + prior_trace: list[dict] | None, + parent_task_run: TaskRun | None, + ) -> TaskRun: + self.calls.append( + { + "input": input, + "prior_trace": prior_trace, + "parent_task_run": parent_task_run, + } + ) + if not self._replies: + raise AssertionError("FakeInvoker called more times than replies provided") + assistant_reply = self._replies.pop(0) + if prior_trace is None: + new_trace = [ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": input}, + {"role": "assistant", "content": assistant_reply}, + ] + else: + new_trace = [ + *prior_trace, + {"role": "user", "content": input}, + {"role": "assistant", "content": assistant_reply}, + ] + return _fake_run(new_trace, run_id=f"run-turn-{len(self.calls)}") + + +def _su_driver_with_replies(replies: list[str]) -> Mock: + """Mock(spec=SyntheticUserDriver) with respond() returning canned strings.""" + drv = Mock(spec=SyntheticUserDriver) + drv.respond = AsyncMock(side_effect=replies) + return drv + + +# ───────────────────────── happy path ───────────────────────── + + +@pytest.mark.asyncio +async def test_drive_case_runs_exactly_turns_iterations() -> None: + """No early termination — loop always completes `turns` iterations.""" + invoker = _FakeInvoker(assistant_replies=["a1", "a2", "a3", "a4"]) + su = _su_driver_with_replies(["u2", "u3", "u4", "u5"]) + + result = await drive_case( + case=_case(), + target_invoker=invoker, + su_driver=su, + turns=4, + ) + + assert isinstance(result, DriveCaseResult) + assert len(result.chain) == 4 + assert len(invoker.calls) == 4 + assert su.respond.await_count == 4 + + +@pytest.mark.asyncio +async def test_drive_case_seeds_first_turn_with_case_seed_prompt() -> None: + invoker = _FakeInvoker(assistant_replies=["a1"]) + su = _su_driver_with_replies(["next user msg"]) + + await drive_case( + case=_case(seed="custom seed"), + target_invoker=invoker, + su_driver=su, + turns=1, + ) + + first = invoker.calls[0] + assert first["input"] == "custom seed" + assert first["prior_trace"] is None + assert first["parent_task_run"] is None + + +@pytest.mark.asyncio +async def test_drive_case_threads_prior_trace_and_parent_run() -> None: + invoker = _FakeInvoker(assistant_replies=["r1", "r2", "r3"]) + su = _su_driver_with_replies(["u2", "u3", "u4"]) + + result = await drive_case( + case=_case(seed="u1"), + target_invoker=invoker, + su_driver=su, + turns=3, + ) + + # Turn 2's invoke should receive turn 1's trace + TaskRun. + second = invoker.calls[1] + assert second["input"] == "u2" + assert second["prior_trace"] is result.chain[0].trace + assert second["parent_task_run"] is result.chain[0] + + # Turn 3's invoke should receive turn 2's trace + TaskRun. + third = invoker.calls[2] + assert third["input"] == "u3" + assert third["prior_trace"] is result.chain[1].trace + assert third["parent_task_run"] is result.chain[1] + + +@pytest.mark.asyncio +async def test_drive_case_passes_full_trace_to_su_driver() -> None: + """The SU driver's `respond` receives the full cumulative trace — + the driver itself filters to visible_message_roles. + """ + invoker = _FakeInvoker(assistant_replies=["a1", "a2"]) + su = _su_driver_with_replies(["u2", "u3"]) + + result = await drive_case( + case=_case(seed="u1"), + target_invoker=invoker, + su_driver=su, + turns=2, + ) + + # First respond() got turn-1's trace ([sys, u1, a1]). + first_call_args = su.respond.await_args_list[0].args + assert first_call_args[0] == result.chain[0].trace + # Second respond() got turn-2's trace. + second_call_args = su.respond.await_args_list[1].args + assert second_call_args[0] == result.chain[1].trace + + +# ───────────────────────── on_turn hook ───────────────────────── + + +@pytest.mark.asyncio +async def test_drive_case_on_turn_hook_fires_once_per_turn() -> None: + invoker = _FakeInvoker(assistant_replies=["a1", "a2", "a3"]) + su = _su_driver_with_replies(["u2", "u3", "u4"]) + + captured: list[tuple[TaskRun, str]] = [] + + async def _hook(*, run: TaskRun, su_message: str) -> None: + captured.append((run, su_message)) + + result = await drive_case( + case=_case(), + target_invoker=invoker, + su_driver=su, + turns=3, + on_turn=_hook, + ) + + assert len(captured) == 3 + for i, (run, msg) in enumerate(captured): + assert run is result.chain[i] + # SU's replies are u2, u3, u4 per the fake. + assert msg == ["u2", "u3", "u4"][i] + + +@pytest.mark.asyncio +async def test_drive_case_works_without_on_turn_hook() -> None: + """on_turn is optional — the loop runs normally if it's None.""" + invoker = _FakeInvoker(assistant_replies=["a1"]) + su = _su_driver_with_replies(["u2"]) + + result = await drive_case( + case=_case(), + target_invoker=invoker, + su_driver=su, + turns=1, + on_turn=None, + ) + + assert len(result.chain) == 1 + + +# ───────────────────────── invariants ───────────────────────── + + +@pytest.mark.asyncio +async def test_drive_case_rejects_invalid_turns() -> None: + invoker = _FakeInvoker(assistant_replies=[]) + su = _su_driver_with_replies([]) + + with pytest.raises(ValueError, match="turns must be >= 1"): + await drive_case( + case=_case(), + target_invoker=invoker, + su_driver=su, + turns=0, + ) + + +@pytest.mark.asyncio +async def test_drive_case_propagates_target_invoker_errors() -> None: + """If the target_invoker raises, the loop doesn't swallow — the + runner is responsible for per-case isolation, not drive_case. + """ + bad_invoker = AsyncMock(side_effect=RuntimeError("target blew up")) + su = _su_driver_with_replies([]) + + with pytest.raises(RuntimeError, match="target blew up"): + await drive_case( + case=_case(), + target_invoker=bad_invoker, + su_driver=su, + turns=3, + ) + + +@pytest.mark.asyncio +async def test_drive_case_propagates_su_driver_errors() -> None: + """If su_driver.respond raises, the loop doesn't swallow either.""" + invoker = _FakeInvoker(assistant_replies=["a1"]) + su = Mock(spec=SyntheticUserDriver) + su.respond = AsyncMock(side_effect=ValueError("bad conversation")) + + with pytest.raises(ValueError, match="bad conversation"): + await drive_case( + case=_case(), + target_invoker=invoker, + su_driver=su, + turns=3, + ) + + +# ───────────────────────── chain order ───────────────────────── + + +@pytest.mark.asyncio +async def test_drive_case_returns_chain_in_order_leaf_last() -> None: + invoker = _FakeInvoker(assistant_replies=["a1", "a2", "a3"]) + su = _su_driver_with_replies(["u2", "u3", "u4"]) + + result = await drive_case( + case=_case(seed="u1"), + target_invoker=invoker, + su_driver=su, + turns=3, + ) + + assert [r.id for r in result.chain] == ["run-turn-1", "run-turn-2", "run-turn-3"] + # Leaf's trace is the longest — covers all three round-trips. + assert len(result.chain[-1].trace) > len(result.chain[0].trace) diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/app/desktop/studio_server/synthetic_user/test_runner.py new file mode 100644 index 0000000000..4de3e05505 --- /dev/null +++ b/app/desktop/studio_server/synthetic_user/test_runner.py @@ -0,0 +1,576 @@ +"""Unit tests for run_cases_batch. + +`adapter_for_task` is monkeypatched to return a fake adapter whose +`.invoke` is an AsyncMock; `SyntheticUserDriver` is replaced with a +patched class returning a mocked instance whose `respond()` yields +canned strings. Tests collect the event stream into a list and inspect +ordering / per-case bookkeeping. +""" + +import asyncio +from typing import Any +from unittest.mock import AsyncMock, Mock + +import pytest + +from kiln_ai.datamodel.datamodel_enums import ( + ModelProviderName, + StructuredOutputMode, +) +from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig +from kiln_ai.datamodel.task import Task +from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.driver import SyntheticUserDriver +from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig +from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) +from app.desktop.studio_server.synthetic_user import runner as runner_mod +from app.desktop.studio_server.synthetic_user.runner import ( + BatchCompletedEvent, + BatchEvent, + BatchStartedEvent, + CaseCompletedEvent, + CaseFailedEvent, + TurnCompletedEvent, + run_cases_batch, +) + + +# ───────────────────────── helpers / fixtures ───────────────────────── + + +def _case(idx: int = 0) -> SyntheticUserCase: + return SyntheticUserCase( + seed_prompt=f"seed-{idx}", + synthetic_user_info=( + f"persona-{idx}" + f"goal-{idx}" + f"guidance-{idx}" + ), + ) + + +def _su_driver_config() -> SyntheticUserDriverConfig: + return SyntheticUserDriverConfig( + model_name="claude_4_5_haiku", + model_provider_name=ModelProviderName.openrouter, + ) + + +def _target_run_config() -> KilnAgentRunConfigProperties: + return KilnAgentRunConfigProperties( + model_name="gpt_5_5", + model_provider_name=ModelProviderName.openrouter, + prompt_id="simple_prompt_builder", + structured_output_mode=StructuredOutputMode.default, + tools_config=ToolsRunConfig(tools=[]), + ) + + +def _fake_run(run_id: str, cost: float = 0.0) -> Mock: + """Stand-in for a persisted TaskRun. drive_case reads `.trace`; runner + reads `.id` + `.cumulative_usage.cost`; `_tag_leaf` writes `.tags` + and calls `.save_to_file()`. + """ + run = Mock(spec=TaskRun) + run.id = run_id + run.trace = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "u"}, + {"role": "assistant", "content": f"a-{run_id}"}, + ] + run.cumulative_usage = Mock(cost=cost) + run.tags = [] + run.save_to_file = Mock() + return run + + +def _patch_adapter_for_task( + monkeypatch: pytest.MonkeyPatch, invoke_side_effect: Any +) -> AsyncMock: + """Make `adapter_for_task` return an adapter whose .invoke yields the + given side_effect (list of TaskRuns or a callable). + """ + adapter = Mock() + adapter.invoke = AsyncMock(side_effect=invoke_side_effect) + monkeypatch.setattr( + runner_mod, "adapter_for_task", lambda task, run_config: adapter + ) + return adapter.invoke + + +def _patch_su_driver( + monkeypatch: pytest.MonkeyPatch, replies_per_case: dict[int, list[str]] | list[str] +) -> None: + """Replace SyntheticUserDriver with a stub that returns canned strings. + + Pass a list to give every case the same reply schedule; pass a dict + keyed by case_index for per-case schedules. + """ + call_counter = {"i": 0} + + def _ctor(blob, config): # noqa: ARG001 + idx = call_counter["i"] + call_counter["i"] += 1 + replies = ( + replies_per_case[idx] + if isinstance(replies_per_case, dict) + else list(replies_per_case) + ) + instance = Mock(spec=SyntheticUserDriver) + instance.respond = AsyncMock(side_effect=replies) + return instance + + monkeypatch.setattr(runner_mod, "SyntheticUserDriver", _ctor) + + +def _patch_su_driver_factory(monkeypatch: pytest.MonkeyPatch, factory: Any) -> None: + """For when tests need full control (e.g., raise on construction).""" + monkeypatch.setattr(runner_mod, "SyntheticUserDriver", factory) + + +@pytest.fixture +def fake_task() -> Mock: + return Mock(spec=Task) + + +async def _collect(gen) -> list[BatchEvent]: + out: list[BatchEvent] = [] + async for ev in gen: + out.append(ev) + return out + + +# ───────────────────────── input validation ───────────────────────── + + +@pytest.mark.asyncio +async def test_empty_cases_raises(fake_task: Mock) -> None: + with pytest.raises(ValueError, match="cases cannot be empty"): + async for _ in run_cases_batch( + cases=[], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + ): + pass + + +@pytest.mark.asyncio +async def test_invalid_turns_raises(fake_task: Mock) -> None: + with pytest.raises(ValueError, match="turns must be >= 1"): + async for _ in run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=0, + ): + pass + + +@pytest.mark.asyncio +async def test_invalid_concurrency_raises(fake_task: Mock) -> None: + with pytest.raises(ValueError, match="concurrency must be >= 1"): + async for _ in run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + concurrency=0, + ): + pass + + +# ───────────────────────── happy path ───────────────────────── + + +@pytest.mark.asyncio +async def test_three_cases_produce_full_event_stream( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + cases = [_case(0), _case(1), _case(2)] + _patch_adapter_for_task( + monkeypatch, + [ + _fake_run("r0", cost=0.01), + _fake_run("r1", cost=0.02), + _fake_run("r2", cost=0.04), + ], + ) + _patch_su_driver(monkeypatch, replies_per_case=["u-next"]) + + events = await _collect( + run_cases_batch( + cases=cases, + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + concurrency=4, + batch_tag="testbatch", + ) + ) + + assert isinstance(events[0], BatchStartedEvent) + assert events[0].batch_tag == "testbatch" + assert events[0].num_cases == 3 + + assert isinstance(events[-1], BatchCompletedEvent) + assert events[-1].successful == 3 + assert events[-1].failed == 0 + assert events[-1].batch_tag == "testbatch" + assert events[-1].total_cost == pytest.approx(0.07) + + turn_events = [e for e in events if isinstance(e, TurnCompletedEvent)] + case_done = [e for e in events if isinstance(e, CaseCompletedEvent)] + assert len(turn_events) == 3 + assert len(case_done) == 3 + assert {e.case_index for e in case_done} == {0, 1, 2} + for ev in case_done: + assert ev.total_turns == 1 + + +@pytest.mark.asyncio +async def test_turn_completed_event_carries_su_message_and_trace( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + _patch_adapter_for_task(monkeypatch, [_fake_run("r0", cost=0.01)]) + _patch_su_driver(monkeypatch, replies_per_case=["the SU's reply"]) + + events = await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + ) + + turn = next(e for e in events if isinstance(e, TurnCompletedEvent)) + assert turn.su_next_message == "the SU's reply" + assert turn.cumulative_cost == pytest.approx(0.01) + # Trace is whatever the fake run carried. + assert any(m.get("role") == "assistant" for m in turn.trace) + + +@pytest.mark.asyncio +async def test_leaf_is_tagged_with_synthetic_user_case_and_batch_tag( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + leaf = _fake_run("leaf") + _patch_adapter_for_task(monkeypatch, [leaf]) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + batch_tag="abc123", + ) + ) + + assert "synthetic_user_case" in leaf.tags + assert "synthetic_user_batch:abc123" in leaf.tags + leaf.save_to_file.assert_called_once() + + +@pytest.mark.asyncio +async def test_auto_generates_batch_tag_when_not_provided( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + _patch_adapter_for_task(monkeypatch, [_fake_run("r")]) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + events = await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + ) + + started = next(e for e in events if isinstance(e, BatchStartedEvent)) + assert len(started.batch_tag) == 12 + assert all(c in "0123456789abcdef" for c in started.batch_tag) + + +# ───────────────────────── per-case failure isolation ───────────────────────── + + +@pytest.mark.asyncio +async def test_malformed_blob_surfaces_as_case_failed( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """SyntheticUserDriver construction raises on a bad blob — that case + fails alone; the others run normally. + """ + call_counter = {"i": 0} + + def _ctor(blob, config): # noqa: ARG001 + idx = call_counter["i"] + call_counter["i"] += 1 + if idx == 1: + raise SyntheticUserInfoParseError("bad blob") + instance = Mock(spec=SyntheticUserDriver) + instance.respond = AsyncMock(return_value="ok") + return instance + + _patch_su_driver_factory(monkeypatch, _ctor) + _patch_adapter_for_task(monkeypatch, [_fake_run("r0"), _fake_run("r2")]) + + events = await _collect( + run_cases_batch( + cases=[_case(0), _case(1), _case(2)], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + concurrency=1, # serialize so the SU driver factory is called in case order + ) + ) + + case_done = [e for e in events if isinstance(e, CaseCompletedEvent)] + case_failed = [e for e in events if isinstance(e, CaseFailedEvent)] + assert {e.case_index for e in case_done} == {0, 2} + assert {e.case_index for e in case_failed} == {1} + assert case_failed[0].error_code == "bad_synthetic_user_info" + + +@pytest.mark.asyncio +async def test_target_invoke_failure_surfaces_as_case_failed( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """If adapter.invoke raises, the case is marked failed — but other + in-flight cases keep running. + """ + _patch_adapter_for_task(monkeypatch, RuntimeError("kaboom")) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + events = await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + ) + + failed = next(e for e in events if isinstance(e, CaseFailedEvent)) + assert failed.error_code == "unexpected_error" + assert "RuntimeError" in failed.message + assert "kaboom" in failed.message + + +@pytest.mark.asyncio +async def test_tag_leaf_failure_surfaces_as_case_failed( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """If save_to_file fails (e.g., disk full, validator rejects the tag), + the case becomes case_failed instead of silently vanishing. + """ + bad_leaf = _fake_run("bad-leaf") + bad_leaf.save_to_file = Mock(side_effect=OSError("disk full")) + _patch_adapter_for_task(monkeypatch, [bad_leaf]) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + events = await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + ) + + failed = next(e for e in events if isinstance(e, CaseFailedEvent)) + assert failed.error_code == "unexpected_error" + assert "disk full" in failed.message + + +# ───────────────────────── concurrency ───────────────────────── + + +@pytest.mark.asyncio +async def test_concurrency_semaphore_caps_in_flight_cases( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """With concurrency=2 and 5 cases, max-in-flight should be exactly 2.""" + in_flight = 0 + max_seen = 0 + fan_out_event = asyncio.Event() + lock = asyncio.Lock() + + async def slow_invoke(**_kwargs: Any) -> Mock: + nonlocal in_flight, max_seen + async with lock: + in_flight += 1 + max_seen = max(max_seen, in_flight) + if in_flight >= 2: + fan_out_event.set() + # Block until fan-out has been observed to actually happen. + try: + await asyncio.wait_for(fan_out_event.wait(), timeout=1.0) + except asyncio.TimeoutError: + pass + async with lock: + in_flight -= 1 + return _fake_run("r") + + _patch_adapter_for_task(monkeypatch, slow_invoke) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + await _collect( + run_cases_batch( + cases=[_case(i) for i in range(5)], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + concurrency=2, + ) + ) + + # Exactly 2 — not <=2 — so a regression that serialized everything + # wouldn't pass vacuously. + assert max_seen == 2 + + +# ───────────────────────── input_source attribution ───────────────────── + + +@pytest.mark.asyncio +async def test_root_input_source_carries_blob_and_seed_prompt( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """First adapter.invoke call gets an input_source with the full SU + case context: the opaque blob + seed_prompt. + """ + captured: list[dict[str, Any]] = [] + + async def _capture(**kwargs: Any) -> Mock: + captured.append(kwargs) + return _fake_run(f"r-{len(captured)}") + + _patch_adapter_for_task(monkeypatch, _capture) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + case = _case(0) + await _collect( + run_cases_batch( + cases=[case], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + batch_tag="rb1", + ) + ) + + root_invoke = captured[0] + props = root_invoke["input_source"].properties + assert props["adapter_name"] == "kiln_synthetic_user_runner" + assert props["model_name"] == "claude_4_5_haiku" + assert props["model_provider"] == "openrouter" + assert props["batch_tag"] == "rb1" + assert props["turn_index"] == 1 + assert props["synthetic_user_info"] == case.synthetic_user_info + assert props["seed_prompt"] == case.seed_prompt + + +@pytest.mark.asyncio +async def test_non_root_input_source_is_slim( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """Second turn's input_source carries only batch_tag + turn_index; + the case context lives on the root. + """ + captured: list[dict[str, Any]] = [] + + async def _capture(**kwargs: Any) -> Mock: + captured.append(kwargs) + return _fake_run(f"r-{len(captured)}") + + _patch_adapter_for_task(monkeypatch, _capture) + _patch_su_driver(monkeypatch, replies_per_case=["u2", "u3"]) + + await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=2, + batch_tag="rb2", + ) + ) + + assert len(captured) == 2 + second_props = captured[1]["input_source"].properties + assert "synthetic_user_info" not in second_props + assert "seed_prompt" not in second_props + assert second_props["batch_tag"] == "rb2" + assert second_props["turn_index"] == 2 + + +# ───────────────────────── consumer cancellation ───────────────────────── + + +@pytest.mark.asyncio +async def test_consumer_cancellation_cancels_in_flight_case_tasks( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """If the consumer breaks off mid-stream, the closer cancels in-flight + case tasks rather than letting them run to completion writing to a + dead queue. + """ + case_started = asyncio.Event() + case_can_proceed = asyncio.Event() + saw_cancel = {"cancelled": False} + + async def _slow_invoke(**_kwargs: Any) -> Mock: + case_started.set() + try: + await case_can_proceed.wait() + except asyncio.CancelledError: + saw_cancel["cancelled"] = True + raise + return _fake_run("r") + + _patch_adapter_for_task(monkeypatch, _slow_invoke) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + gen = run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + + # Drain BatchStartedEvent, then break. + started = await gen.__anext__() + assert isinstance(started, BatchStartedEvent) + + # Wait until the case task is actually running. + await asyncio.wait_for(case_started.wait(), timeout=1.0) + + # Close the generator — simulates consumer disconnect. This should + # cancel the in-flight case task via the finally block. + await gen.aclose() + + # Give the cancellation a beat to propagate. + await asyncio.sleep(0) + + assert saw_cancel["cancelled"] is True From e5894cc8c81f86c0dd55037b667d16e2b4d19d11 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:44:12 -0700 Subject: [PATCH 07/36] feat(studio_server): multiturn_sdg FastAPI routes for SU eval data Two routes for the multi-turn synthetic-user data-generation pipeline: - POST .../multiturn_sdg/generate_cases (sync JSON) - POST .../multiturn_sdg/run_cases_batch (SSE via CancellableStreamingResponse) Wires connect_multiturn_sdg_api into desktop_server.make_app and registers the Multiturn SDG tag in kiln_server's tags_metadata so the regenerated api_schema.d.ts surfaces the routes in the typed client. Both routes guard task.turn_mode == multiturn before doing any upstream work and route SyntheticUserClient typed errors through to faithful HTTP statuses (401/422/502 preserved, not collapsed). The SSE route threads build_save_context(request) into run_cases_batch and uses an isinstance whitelist on the JSON encoder so future Pydantic types on the wire need explicit review. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/desktop/desktop_server.py | 2 + .../studio_server/multiturn_sdg_api.py | 404 ++++++++++ .../studio_server/test_multiturn_sdg_api.py | 735 ++++++++++++++++++ app/web_ui/src/lib/api_schema.d.ts | 175 +++++ libs/server/kiln_server/server.py | 4 + 5 files changed, 1320 insertions(+) create mode 100644 app/desktop/studio_server/multiturn_sdg_api.py create mode 100644 app/desktop/studio_server/test_multiturn_sdg_api.py diff --git a/app/desktop/desktop_server.py b/app/desktop/desktop_server.py index 0163c146c2..e3a6505b33 100644 --- a/app/desktop/desktop_server.py +++ b/app/desktop/desktop_server.py @@ -33,6 +33,7 @@ from app.desktop.studio_server.eval_api import connect_evals_api from app.desktop.studio_server.finetune_api import connect_fine_tune_api from app.desktop.studio_server.import_api import connect_import_api +from app.desktop.studio_server.multiturn_sdg_api import connect_multiturn_sdg_api from app.desktop.studio_server.prompt_api import connect_prompt_api from app.desktop.studio_server.prompt_optimization_job_api import ( connect_prompt_optimization_job_api, @@ -138,6 +139,7 @@ def make_app(tk_root: tk.Tk | None = None): connect_skill_api(app) connect_prompt_optimization_job_api(app) connect_copilot_api(app) + connect_multiturn_sdg_api(app) connect_git_sync_api(app) connect_agent_api(app) connect_dev_tools(app) diff --git a/app/desktop/studio_server/multiturn_sdg_api.py b/app/desktop/studio_server/multiturn_sdg_api.py new file mode 100644 index 0000000000..524920ba11 --- /dev/null +++ b/app/desktop/studio_server/multiturn_sdg_api.py @@ -0,0 +1,404 @@ +"""FastAPI routes for multi-turn synthetic data generation. + +Two routes wrap the runner so the web UI can drive it without a +Python REPL: + + POST /api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/generate_cases + Synchronous JSON. Calls kiln_server `/generate` via the local + SyntheticUserClient and returns the N cases as the SDK shape + (`{seed_prompt, synthetic_user_info: }` per case). + + POST /api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch + SSE stream. Takes (possibly edited) cases + run config + SU driver + config, runs the drive loop concurrently across cases, and emits + BatchEvent frames as `data:` lines. Terminator is + `data: complete\\n\\n`, matching the eval_api SSE convention. + +Both routes guard `task.turn_mode == TurnMode.multiturn` before doing any +upstream work — the runner depends on multi-turn TaskRun chaining +(parent_task_run_id is rejected on single-turn tasks). + +The kiln_server API key is read server-side (`get_copilot_api_key`) and +never crosses to the browser, matching the copilot pattern. The SU +driver model is exposed to the caller because the choice of model +affects probe quality and cost — deliberate, not internal. +""" + +import dataclasses +import json +import logging +from typing import Annotated, Any + +from fastapi import FastAPI, HTTPException, Path, Request +from fastapi.responses import StreamingResponse +from kiln_ai.datamodel.datamodel_enums import ( + ModelProviderName, + StructuredOutputMode, + TurnMode, +) +from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig +from kiln_ai.datamodel.task import Task +from kiln_ai.datamodel.usage import MessageUsage +from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig +from kiln_server.cancellable_streaming_response import CancellableStreamingResponse +from kiln_server.git_sync_decorators import build_save_context, no_write_lock +from kiln_server.task_api import task_from_id +from kiln_server.utils.agent_checks.policy import agent_policy_require_approval +from pydantic import BaseModel, Field + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) +from app.desktop.studio_server.synthetic_user.client import ( + SyntheticUserClient, + SyntheticUserRequestError, + SyntheticUserServerError, +) +from app.desktop.studio_server.synthetic_user.runner import ( + CONCURRENCY, + DEFAULT_TURNS, + NUM_CASES_MAX, + BatchCompletedEvent, + BatchEvent, + BatchStartedEvent, + CaseCompletedEvent, + CaseFailedEvent, + TurnCompletedEvent, + run_cases_batch, +) +from app.desktop.studio_server.utils.copilot_utils import get_copilot_api_key + +logger = logging.getLogger(__name__) + + +# ───────────────────────── Pydantic API models ───────────────────────── + +# Cases ride the wire as `list[dict[str, Any]]` rather than a Pydantic +# mirror — the SDK's SyntheticUserCase is the single source of truth, and +# we round-trip via `to_dict` / `from_dict` at the boundary. Trade-off: +# TS bindings type cases as `Record` instead of getting +# per-field autocomplete. +SyntheticUserCaseDict = dict[str, Any] +_CASE_DICT_DESCRIPTION = ( + "A SyntheticUserCase as returned by kiln_server's /generate. Shape: " + "{seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info " + "value is an XML-tagged blob: " + ".......... " + "Parsed client-side by kiln_ai.synthetic_user.parser." +) + + +class GenerateCasesApiInput(BaseModel): + target_specification: str = Field(..., min_length=1) + num_cases: int = Field(..., ge=1, le=NUM_CASES_MAX) + + +class GenerateCasesApiOutput(BaseModel): + cases: list[SyntheticUserCaseDict] = Field(..., description=_CASE_DICT_DESCRIPTION) + + +class TargetRunConfigSpec(BaseModel): + """How to invoke the target task on each turn — same fields a manual + UI run would use. + """ + + model_name: str = Field(..., min_length=1) + model_provider: ModelProviderName + prompt_id: str = Field(default="simple_prompt_builder", min_length=1) + + +class SyntheticUserDriverSpec(BaseModel): + """How to drive the synthetic user. Caller controls because probe + quality and cost both depend on the model. + """ + + model_name: str = Field(..., min_length=1) + model_provider: ModelProviderName + + +class RunCasesBatchApiInput(BaseModel): + cases: list[SyntheticUserCaseDict] = Field( + ..., + min_length=1, + max_length=NUM_CASES_MAX, + description=( + f"Cases as returned by /generate_cases, optionally edited. " + f"{_CASE_DICT_DESCRIPTION}" + ), + ) + turns: int = Field( + default=DEFAULT_TURNS, + ge=1, + le=20, + description=( + "Exact number of assistant turns to produce per case. The drive " + "loop has no early termination." + ), + ) + target_run_config: TargetRunConfigSpec + su_driver: SyntheticUserDriverSpec + batch_tag: str | None = Field( + default=None, + pattern=r"^[A-Za-z0-9_-]+$", + min_length=1, + max_length=64, + description=( + "Optional user-supplied batch label. Constrained to " + "[A-Za-z0-9_-]{1,64} so it can safely be used as a tag on leaf " + "TaskRuns. Auto-generated if not provided." + ), + ) + + +# ───────────────────────── helpers ───────────────────────── + + +def _guard_multiturn(task: Task) -> None: + """Reject early if the caller pointed us at a single-turn task. The + runner's chained TaskRun shape (parent_task_run_id) is rejected on + single-turn tasks by the datamodel validator — better to surface a + clean 400 here than a mid-stream chain corruption. + """ + if task.turn_mode != TurnMode.multiturn: + raise HTTPException( + status_code=400, + detail={ + "code": "task_not_multiturn", + "message": ( + "Multi-turn synthetic data generation requires a task with " + "turn_mode=multiturn." + ), + }, + ) + + +def _to_target_run_config(spec: TargetRunConfigSpec) -> KilnAgentRunConfigProperties: + return KilnAgentRunConfigProperties( + model_name=spec.model_name, + model_provider_name=spec.model_provider, + prompt_id=spec.prompt_id, + structured_output_mode=StructuredOutputMode.default, + tools_config=ToolsRunConfig(tools=[]), + ) + + +def _to_su_driver_config(spec: SyntheticUserDriverSpec) -> SyntheticUserDriverConfig: + return SyntheticUserDriverConfig( + model_name=spec.model_name, + model_provider_name=spec.model_provider, + ) + + +# Maps each event dataclass to the snake_case `event` discriminator on the +# SSE frame. Keeps the wire shape stable even if the dataclass types are +# renamed later. +_EVENT_NAMES: dict[type, str] = { + BatchStartedEvent: "batch_started", + TurnCompletedEvent: "turn_completed", + CaseCompletedEvent: "case_completed", + CaseFailedEvent: "case_failed", + BatchCompletedEvent: "batch_completed", +} + + +def _event_to_payload(event: BatchEvent) -> dict: + name = _EVENT_NAMES.get(type(event)) + if name is None: + # New dataclass added without registering it — fail loud rather + # than silently swallowing. + raise RuntimeError(f"Unregistered BatchEvent type: {type(event).__name__}") + return {"event": name, **dataclasses.asdict(event)} + + +def _jsonable(obj: Any) -> Any: + """json.dumps `default` handler. SSE trace frames embed `MessageUsage` + (Pydantic) on assistant turns, which doesn't survive `dataclasses.asdict` + recursion. The whitelist is intentionally narrow: any new Pydantic type + on the wire must be added here explicitly, prompting a review for + whether `model_dump()` exposes sensitive fields. Do NOT broaden to + `hasattr(obj, "model_dump")` or `str(obj)` — silent leakage is worse + than a loud TypeError that fails the stream. + """ + if isinstance(obj, MessageUsage): + return obj.model_dump() + raise TypeError(f"{type(obj).__name__} is not JSON serializable") + + +def _to_http_exception(exc: Exception) -> HTTPException: + """Translate SyntheticUserClient's typed exceptions to HTTPExceptions. + + Returns the exception rather than raising so callers can `raise … from exc` + at the call site — gives the type checker NoReturn semantics for free + and avoids any chance of unbound-variable bugs after the try block. + + Status preservation: upstream's HTTP status is passed through faithfully + when it's one of the 4xx/5xx codes the SDK models (401, 422, 500, 502). + Collapsing a 401 to 400 hides whether the operator's stored API key is + bad vs the caller's body being malformed — both knowable distinctions + that the consumer needs to act on. + """ + if isinstance(exc, SyntheticUserRequestError): + # 401 (kiln_server auth failed) and 422 (runner sent a bad body) + # are both client-class errors but distinct causes; preserve. + status = exc.status_code if exc.status_code in (401, 422) else 400 + return HTTPException( + status_code=status, + detail={"code": exc.code, "message": exc.message}, + ) + if isinstance(exc, SyntheticUserServerError): + # Preserve the upstream 5xx (502 → 502, 503 → 503, ...). Anything + # unrecognized falls to a clean 500. + status = ( + exc.status_code if exc.status_code and 500 <= exc.status_code < 600 else 500 + ) + return HTTPException( + status_code=status, + detail={"code": exc.code, "message": exc.message}, + ) + # Generic fallback — shouldn't happen for known typed exceptions but + # keeps the helper exhaustive at the call site. + return HTTPException( + status_code=500, + detail={"code": "internal_error", "message": str(exc)}, + ) + + +# ───────────────────────── route registration ───────────────────────── + + +def connect_multiturn_sdg_api(app: FastAPI) -> None: + @app.post( + "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/generate_cases", + tags=["Multiturn SDG"], + summary="Generate Multi-Turn SU Cases", + openapi_extra=agent_policy_require_approval( + "Generate synthetic-user cases? Uses an LLM call (cost)." + ), + ) + async def generate_cases( + project_id: Annotated[ + str, Path(description="ID of the project containing the target task.") + ], + task_id: Annotated[ + str, + Path( + description=("ID of the target task. Must be a multi-turn task."), + ), + ], + input: GenerateCasesApiInput, + ) -> GenerateCasesApiOutput: + task = task_from_id(project_id, task_id) + _guard_multiturn(task) + + api_key = get_copilot_api_key() + client = SyntheticUserClient(api_key=api_key) + + try: + sdk_cases = await client.generate( + target_task_prompt=task.instruction, + target_specification=input.target_specification, + num_cases=input.num_cases, + ) + except (SyntheticUserRequestError, SyntheticUserServerError) as exc: + raise _to_http_exception(exc) from exc + + return GenerateCasesApiOutput(cases=[c.to_dict() for c in sdk_cases]) + + @app.post( + "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch", + tags=["Multiturn SDG"], + summary="Run Multi-Turn SU Cases Batch", + openapi_extra=agent_policy_require_approval( + "Run a multi-turn synthetic-user batch? Invokes the target model " + "and the SU driver model for several turns per case (cost)." + ), + ) + @no_write_lock + async def stream_run_cases_batch( + request: Request, + project_id: Annotated[ + str, Path(description="ID of the project containing the target task.") + ], + task_id: Annotated[ + str, + Path( + description=("ID of the target task. Must be a multi-turn task."), + ), + ], + input: RunCasesBatchApiInput, + ) -> StreamingResponse: + # Guard + decode happen before the stream opens so the client sees + # a clean 400 / 422 rather than a half-open text/event-stream on + # bad input. + task = task_from_id(project_id, task_id) + _guard_multiturn(task) + + # SDK from_dict raises on missing/wrong-typed required fields; + # surface that as a clean 400 instead of letting it explode inside + # the SSE generator. + try: + sdk_cases = [SyntheticUserCase.from_dict(c) for c in input.cases] + except (KeyError, TypeError, ValueError) as exc: + raise HTTPException( + status_code=400, + detail={ + "code": "invalid_case_shape", + "message": (f"Could not parse cases against the SDK shape: {exc}"), + }, + ) from exc + + target_run_config = _to_target_run_config(input.target_run_config) + su_driver_config = _to_su_driver_config(input.su_driver) + save_context = build_save_context(request) + + async def event_generator(): + try: + async for event in run_cases_batch( + cases=sdk_cases, + target_task=task, + target_run_config=target_run_config, + su_driver_config=su_driver_config, + turns=input.turns, + concurrency=CONCURRENCY, + batch_tag=input.batch_tag, + save_context=save_context, + ): + yield ( + "data: " + + json.dumps( + _event_to_payload(event), + default=_jsonable, + ensure_ascii=False, + ) + + "\n\n" + ) + except Exception as e: # noqa: BLE001 — last-resort surface + # The catch is narrow in practice: run_cases_batch + # swallows per-case failures into CaseFailedEvent, so the + # only paths that escape here are developer bugs + # (RuntimeError from _event_to_payload, TypeError from + # _jsonable). asyncio.CancelledError is BaseException and + # bypasses this except — correct, since cancellation + # means the consumer is gone. + logger.exception("multiturn_sdg run_cases_batch failed mid-stream") + yield ( + "data: " + + json.dumps( + { + "event": "batch_failed", + # Stable wire code; class name goes in message + # so it stays useful for debug without leaking + # internal type names onto the wire contract. + "error_code": "internal_error", + "message": f"{type(e).__name__}: {e}", + }, + ensure_ascii=False, + ) + + "\n\n" + ) + yield "data: complete\n\n" + + return CancellableStreamingResponse( + content=event_generator(), + media_type="text/event-stream", + ) diff --git a/app/desktop/studio_server/test_multiturn_sdg_api.py b/app/desktop/studio_server/test_multiturn_sdg_api.py new file mode 100644 index 0000000000..4b8652c599 --- /dev/null +++ b/app/desktop/studio_server/test_multiturn_sdg_api.py @@ -0,0 +1,735 @@ +"""Tests for the multiturn_sdg FastAPI routes. + +`task_from_id` / `get_copilot_api_key` / `SyntheticUserClient` / +`run_cases_batch` are patched per-test so no real network or filesystem +work happens. For SSE tests we patch `run_cases_batch` to yield canned +BatchEvents and assert the serialized `data:` frames match the expected +event schema. +""" + +import json +from typing import AsyncIterator +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from kiln_server.cancellable_streaming_response import CancellableStreamingResponse + +from kiln_ai.datamodel.datamodel_enums import TurnMode +from kiln_ai.datamodel.task import Task +from kiln_ai.datamodel.usage import MessageUsage +from kiln_server.custom_errors import connect_custom_errors + +from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( + SyntheticUserCase, +) +from app.desktop.studio_server.multiturn_sdg_api import connect_multiturn_sdg_api +from app.desktop.studio_server.synthetic_user.client import ( + SyntheticUserRequestError, + SyntheticUserServerError, +) +from app.desktop.studio_server.synthetic_user.runner import ( + BatchCompletedEvent, + BatchStartedEvent, + CaseCompletedEvent, + CaseFailedEvent, + TurnCompletedEvent, +) + + +# ───────────────────────── fixtures ───────────────────────── + + +@pytest.fixture +def app() -> FastAPI: + # connect_custom_errors mirrors production: kiln_server.make_app() registers a + # global HTTPException handler that rewraps `detail` as `{"message": detail}`, + # so our structured `{"code", "message"}` detail dicts arrive on the wire as + # `{"message": {"code": ..., "message": ...}}`. Tests must mount the same + # handler or they assert a wire shape production never ships. + app = FastAPI() + connect_custom_errors(app) + connect_multiturn_sdg_api(app) + return app + + +@pytest.fixture +def client(app: FastAPI) -> TestClient: + return TestClient(app) + + +def _multiturn_task() -> Mock: + task = Mock(spec=Task) + task.name = "support_agent" + task.instruction = "You are a customer support agent." + task.turn_mode = TurnMode.multiturn + return task + + +def _single_turn_task() -> Mock: + task = Mock(spec=Task) + task.name = "single_turn_task" + task.instruction = "Do one thing." + task.turn_mode = TurnMode.single_turn + return task + + +@pytest.fixture +def patch_task_from_id(): + with patch("app.desktop.studio_server.multiturn_sdg_api.task_from_id") as m: + yield m + + +@pytest.fixture +def patch_api_key(): + with patch( + "app.desktop.studio_server.multiturn_sdg_api.get_copilot_api_key", + return_value="test-key", + ): + yield + + +def _sdk_cases(n: int) -> list[SyntheticUserCase]: + return [ + SyntheticUserCase( + seed_prompt=f"seed-{i}", + synthetic_user_info=( + f"persona-{i}" + f"goal-{i}" + f"guide-{i}" + ), + ) + for i in range(n) + ] + + +def _generate_cases_body(num: int = 3) -> dict: + return { + "target_specification": "agent waives policy under pressure", + "num_cases": num, + } + + +def _run_cases_batch_body(num: int = 3) -> dict: + return { + "cases": [ + { + "seed_prompt": f"seed-{i}", + "synthetic_user_info": ( + f"persona-{i}" + f"goal-{i}" + f"guide-{i}" + ), + } + for i in range(num) + ], + "turns": 3, + "target_run_config": { + "model_name": "gpt_5_5", + "model_provider": "openrouter", + }, + "su_driver": { + "model_name": "claude_4_5_haiku", + "model_provider": "openrouter", + }, + "batch_tag": "testbatch", + } + + +def _parse_sse(response_text: str) -> list[dict | str]: + """SSE → list of decoded events. JSON frames → dict; `complete` → string.""" + events: list[dict | str] = [] + for line in response_text.splitlines(): + if not line.startswith("data: "): + continue + payload = line[len("data: ") :] + if payload == "complete": + events.append("complete") + continue + events.append(json.loads(payload)) + return events + + +# ───────────────────────── generate_cases ───────────────────────── + + +def test_generate_cases_happy_path( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock(return_value=_sdk_cases(3)) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(num=3), + ) + + assert resp.status_code == 200 + body = resp.json() + assert len(body["cases"]) == 3 + # Cases ride the wire as the SDK shape (seed_prompt + opaque blob). + case0 = body["cases"][0] + assert case0["seed_prompt"] == "seed-0" + assert "persona-0" in case0["synthetic_user_info"] + + +def test_generate_cases_rejects_single_turn_task_with_400( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _single_turn_task() + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 400 + assert resp.json()["message"]["code"] == "task_not_multiturn" + + +def test_generate_cases_does_not_call_upstream_when_guard_fails( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _single_turn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + MockClient.assert_not_called() + + +def test_generate_cases_server_error_surfaces_with_status( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """SyntheticUserServerError with status_code=502 → 502 response.""" + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock( + side_effect=SyntheticUserServerError( + "llm_unavailable", "upstream timed out", status_code=502 + ) + ) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 502 + assert resp.json()["message"]["code"] == "llm_unavailable" + + +def test_generate_cases_request_error_surfaces_as_400( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock( + side_effect=SyntheticUserRequestError( + "unsupported_model", "no such model", status_code=400 + ) + ) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 400 + assert resp.json()["message"]["code"] == "unsupported_model" + + +def test_generate_cases_validates_num_cases_upper_bound( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """NUM_CASES_MAX=10 — Pydantic should reject 11 before reaching the body.""" + patch_task_from_id.return_value = _multiturn_task() + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json={"target_specification": "spec", "num_cases": 11}, + ) + assert resp.status_code == 422 + + +# ───────────────────────── run_cases_batch (SSE) ───────────────────────── + + +def _sse_get(client: TestClient, body: dict | None = None): + body = body if body is not None else _run_cases_batch_body() + return client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + + +def test_run_cases_batch_rejects_single_turn_task( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _single_turn_task() + resp = _sse_get(client) + assert resp.status_code == 400 + assert resp.json()["message"]["code"] == "task_not_multiturn" + + +def test_run_cases_batch_emits_full_sse_event_stream( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """Canned BatchEvent sequence → assert wire shape matches.""" + patch_task_from_id.return_value = _multiturn_task() + + canned: list = [ + BatchStartedEvent(batch_tag="testbatch", num_cases=2), + TurnCompletedEvent( + case_index=0, + assistant_run_id="r0a", + su_next_message="next user msg", + cumulative_cost=0.01, + trace=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello back"}, + ], + ), + CaseCompletedEvent( + case_index=0, + chain_run_ids=["r0a"], + leaf_run_id="r0a", + total_turns=1, + target_total_cost=0.01, + ), + CaseFailedEvent( + case_index=1, + error_code="bad_synthetic_user_info", + message="missing required tag", + ), + BatchCompletedEvent( + successful=1, failed=1, batch_tag="testbatch", target_total_cost=0.01 + ), + ] + + async def _fake_runner(**_kwargs) -> AsyncIterator: + for ev in canned: + yield ev + + with patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _fake_runner, + ): + resp = _sse_get(client) + + assert resp.status_code == 200 + assert resp.headers["content-type"].startswith("text/event-stream") + + events = _parse_sse(resp.text) + assert len(events) == 6 # 5 canned events + `complete` terminator + assert events[-1] == "complete" + + assert events[0] == { + "event": "batch_started", + "batch_tag": "testbatch", + "num_cases": 2, + } + assert events[1]["event"] == "turn_completed" + assert events[1]["case_index"] == 0 + assert events[1]["su_next_message"] == "next user msg" + assert events[1]["trace"][2]["content"] == "hello back" + # No stop_signal field on TurnCompletedEvent anymore. + assert "stop_signal" not in events[1] + + assert events[2]["event"] == "case_completed" + # No stop_reason field on CaseCompletedEvent anymore. + assert "stop_reason" not in events[2] + + assert events[3] == { + "event": "case_failed", + "case_index": 1, + "error_code": "bad_synthetic_user_info", + "message": "missing required tag", + } + assert events[4] == { + "event": "batch_completed", + "successful": 1, + "failed": 1, + "batch_tag": "testbatch", + "target_total_cost": 0.01, + } + + +def test_run_cases_batch_jsonable_handles_message_usage_in_trace( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """A real-shape trace can carry MessageUsage Pydantic instances on + assistant turns; `_jsonable` must turn those into JSON without + crashing. + """ + patch_task_from_id.return_value = _multiturn_task() + + usage = MessageUsage(input_tokens=10, output_tokens=20, total_tokens=30, cost=0.001) + canned = [ + BatchStartedEvent(batch_tag="tb", num_cases=1), + TurnCompletedEvent( + case_index=0, + assistant_run_id="r0", + su_next_message="hello", + cumulative_cost=0.001, + trace=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hi back", "usage": usage}, # type: ignore[typeddict-unknown-key] + ], + ), + BatchCompletedEvent( + successful=1, failed=0, batch_tag="tb", target_total_cost=0.001 + ), + ] + + async def _fake_runner(**_kwargs) -> AsyncIterator: + for ev in canned: + yield ev + + with patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _fake_runner, + ): + resp = _sse_get(client) + + assert resp.status_code == 200 + events = _parse_sse(resp.text) + turn = next( + e for e in events if isinstance(e, dict) and e.get("event") == "turn_completed" + ) + # MessageUsage went through .model_dump() — the assistant turn's + # `usage` key is now a plain dict, not a Pydantic instance. + usage_payload = turn["trace"][1]["usage"] + assert isinstance(usage_payload, dict) + assert usage_payload["cost"] == 0.001 + assert usage_payload["input_tokens"] == 10 + + +def test_run_cases_batch_translates_runner_failure_to_batch_failed( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """If the runner raises mid-stream (developer bug), the stream still + terminates cleanly with batch_failed → complete. + """ + patch_task_from_id.return_value = _multiturn_task() + + async def _exploding_runner(**_kwargs) -> AsyncIterator: + raise RuntimeError("upstream catastrophe") + yield # pragma: no cover — unreachable; marks this as a generator + + with patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _exploding_runner, + ): + resp = _sse_get(client) + + assert resp.status_code == 200 + events = _parse_sse(resp.text) + assert events[-1] == "complete" + failed_evt = next( + e for e in events if isinstance(e, dict) and e.get("event") == "batch_failed" + ) + # Stable wire code; class name is in the message for debug, not on + # the contract. + assert failed_evt["error_code"] == "internal_error" + assert "RuntimeError" in failed_evt["message"] + assert "upstream catastrophe" in failed_evt["message"] + + +def test_run_cases_batch_jsonable_typeerror_surfaces_as_batch_failed( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """A non-serializable, non-Pydantic object in trace must not corrupt + the stream — `_jsonable` raises TypeError, the outer except converts + to `batch_failed`. Locks in the fail-loud branch so a future widening + of `_jsonable` (e.g., a defensive `str(obj)` fallback) doesn't sneak + arbitrary content onto the wire. + """ + patch_task_from_id.return_value = _multiturn_task() + + class _NonSerializable: + pass + + canned = [ + BatchStartedEvent(batch_tag="tb", num_cases=1), + TurnCompletedEvent( + case_index=0, + assistant_run_id="r0", + su_next_message="x", + cumulative_cost=0.0, + trace=[ + {"role": "user", "content": "hi"}, + # Slip a non-Pydantic, non-JSON-native object into trace. + { + "role": "assistant", + "content": "hi back", + "usage": _NonSerializable(), + }, # type: ignore[typeddict-unknown-key] + ], + ), + ] + + async def _fake_runner(**_kwargs) -> AsyncIterator: + for ev in canned: + yield ev + + with patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _fake_runner, + ): + resp = _sse_get(client) + + events = _parse_sse(resp.text) + failed_evt = next( + e for e in events if isinstance(e, dict) and e.get("event") == "batch_failed" + ) + assert failed_evt["error_code"] == "internal_error" + assert "TypeError" in failed_evt["message"] + + +def test_run_cases_batch_validates_empty_cases( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _multiturn_task() + body = _run_cases_batch_body() + body["cases"] = [] + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + assert resp.status_code == 422 + + +def test_run_cases_batch_validates_too_many_cases( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + patch_task_from_id.return_value = _multiturn_task() + body = _run_cases_batch_body(num=11) + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + assert resp.status_code == 422 + + +def test_run_cases_batch_rejects_malformed_case_shape_with_400( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """A case missing the synthetic_user_info field → 400, not a half-open stream.""" + patch_task_from_id.return_value = _multiturn_task() + body = _run_cases_batch_body() + body["cases"] = [{"seed_prompt": "hi"}] # missing synthetic_user_info + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + assert resp.status_code == 400 + assert resp.json()["message"]["code"] == "invalid_case_shape" + + +@pytest.mark.parametrize( + "bad_tag", + [ + "", # min_length=1 + "my run", # space + "tag:with:colons", # colon — tag-unsafe (delimiter in our prefix scheme) + "weird*chars", # punctuation + "x" * 65, # max_length=64 + ], +) +def test_run_cases_batch_rejects_invalid_batch_tags( + bad_tag: str, + client: TestClient, + patch_task_from_id, + patch_api_key, +) -> None: + """The batch_tag must be `[A-Za-z0-9_-]{1,64}` — character class and + length boundaries both enforced. Locks in the rule so a future + loosening (e.g., adding `:`) doesn't slip past test coverage. + """ + patch_task_from_id.return_value = _multiturn_task() + body = _run_cases_batch_body() + body["batch_tag"] = bad_tag + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "good_tag", + [ + "a", # min boundary + "x" * 64, # max boundary + "abc-def_123", # hyphen + underscore + alphanumerics + "ABC123", # uppercase + ], +) +def test_run_cases_batch_accepts_valid_batch_tags( + good_tag: str, + client: TestClient, + patch_task_from_id, + patch_api_key, +) -> None: + """Boundary chars that should pass — locks in the accept side of the + pattern so the test pair fully fences the contract. + """ + patch_task_from_id.return_value = _multiturn_task() + + async def _empty_runner(**_kwargs) -> AsyncIterator: + # `if False: yield` keeps this an async generator without ever + # emitting; the route still wraps it in a streaming response. + if False: + yield # pragma: no cover + + body = _run_cases_batch_body() + body["batch_tag"] = good_tag + with patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _empty_runner, + ): + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/run_cases_batch", + json=body, + ) + assert resp.status_code == 200 + + +def test_generate_cases_preserves_upstream_401_status( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """A 401 from kiln_server means our stored API key is bad — surface + as 401, not collapsed to a generic 400. Operator-config problem + surface vs caller-input problem surface should stay distinct. + """ + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock( + side_effect=SyntheticUserRequestError( + "unauthorized", "bad api key", status_code=401 + ) + ) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 401 + assert resp.json()["message"]["code"] == "unauthorized" + + +def test_generate_cases_preserves_upstream_422_status( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """A 422 from kiln_server (runner sent a body the validator rejected) + is a different beast from a caller's-input 400 — preserve. + """ + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock( + side_effect=SyntheticUserRequestError( + "http_422", "body.target_specification: too long", status_code=422 + ) + ) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 422 + assert resp.json()["message"]["code"] == "http_422" + + +def test_run_cases_batch_uses_cancellable_streaming_response( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """Verifies the route wraps its generator in CancellableStreamingResponse. + Without this, browser disconnects don't reach the runner and in-flight + case tasks keep burning LLM calls until they finish. Matches the chat + SSE route's `test_uses_cancellable_streaming_response` discipline. + """ + patch_task_from_id.return_value = _multiturn_task() + + async def _empty_runner(**_kwargs) -> AsyncIterator: + if False: + yield # pragma: no cover + + with ( + patch( + "app.desktop.studio_server.multiturn_sdg_api.run_cases_batch", + _empty_runner, + ), + patch( + "app.desktop.studio_server.multiturn_sdg_api.CancellableStreamingResponse", + wraps=CancellableStreamingResponse, + ) as mock_cls, + ): + resp = _sse_get(client) + _ = resp.content + + assert resp.status_code == 200 + mock_cls.assert_called_once() + + +def test_run_cases_batch_has_no_write_lock_decorator(app: FastAPI) -> None: + """The SSE route must be @no_write_lock so the git_sync middleware + doesn't wrap the entire streaming response in one atomic_write + (which would block all other writes for the batch duration). + """ + path = "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch" + for route in app.routes: + if getattr(route, "path", None) == path and "POST" in getattr( + route, "methods", set() + ): + assert getattr(route.endpoint, "_git_sync_no_write_lock", False), ( + f"POST {path} must be @no_write_lock" + ) + return + raise AssertionError(f"POST {path} route not found") + + +def test_generate_cases_preserves_upstream_503_status( + client: TestClient, patch_task_from_id, patch_api_key +) -> None: + """An unexpected 5xx (503) should not silently collapse to 500.""" + patch_task_from_id.return_value = _multiturn_task() + with patch( + "app.desktop.studio_server.multiturn_sdg_api.SyntheticUserClient" + ) as MockClient: + instance = MockClient.return_value + instance.generate = AsyncMock( + side_effect=SyntheticUserServerError( + "http_503", "upstream unavailable", status_code=503 + ) + ) + + resp = client.post( + "/api/projects/proj-1/tasks/task-1/multiturn_sdg/generate_cases", + json=_generate_cases_body(), + ) + + assert resp.status_code == 503 + assert resp.json()["message"]["code"] == "http_503" diff --git a/app/web_ui/src/lib/api_schema.d.ts b/app/web_ui/src/lib/api_schema.d.ts index 2cc2dfbcf5..6d1d7a7850 100644 --- a/app/web_ui/src/lib/api_schema.d.ts +++ b/app/web_ui/src/lib/api_schema.d.ts @@ -2760,6 +2760,40 @@ export interface paths { patch?: never; trace?: never; }; + "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/generate_cases": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Generate Multi-Turn SU Cases */ + post: operations["generate_cases_api_projects__project_id__tasks__task_id__multiturn_sdg_generate_cases_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Run Multi-Turn SU Cases Batch */ + post: operations["stream_run_cases_batch_api_projects__project_id__tasks__task_id__multiturn_sdg_run_cases_batch_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/git_sync/test_access": { parameters: { query?: never; @@ -6581,6 +6615,23 @@ export interface components { [key: string]: components["schemas"]["SampleApi"][]; }; }; + /** GenerateCasesApiInput */ + GenerateCasesApiInput: { + /** Target Specification */ + target_specification: string; + /** Num Cases */ + num_cases: number; + }; + /** GenerateCasesApiOutput */ + GenerateCasesApiOutput: { + /** + * Cases + * @description A SyntheticUserCase as returned by kiln_server's /generate. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. + */ + cases: { + [key: string]: unknown; + }[]; + }; /** GetRagConfigProgressRequest */ GetRagConfigProgressRequest: { /** @@ -8520,6 +8571,29 @@ export interface components { /** Feedback */ feedback: string; }; + /** RunCasesBatchApiInput */ + RunCasesBatchApiInput: { + /** + * Cases + * @description Cases as returned by /generate_cases, optionally edited. A SyntheticUserCase as returned by kiln_server's /generate. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. + */ + cases: { + [key: string]: unknown; + }[]; + /** + * Turns + * @description Exact number of assistant turns to produce per case. The drive loop has no early termination. + * @default 5 + */ + turns: number; + target_run_config: components["schemas"]["TargetRunConfigSpec"]; + su_driver: components["schemas"]["SyntheticUserDriverSpec"]; + /** + * Batch Tag + * @description Optional user-supplied batch label. Constrained to [A-Za-z0-9_-]{1,64} so it can safely be used as a tag on leaf TaskRuns. Auto-generated if not provided. + */ + batch_tag?: string | null; + }; /** * RunChainEntry * @description A single entry in a multi-turn run's conversation chain. @@ -9286,6 +9360,16 @@ export interface components { /** Prompt */ prompt: string; }; + /** + * SyntheticUserDriverSpec + * @description How to drive the synthetic user. Caller controls because probe + * quality and cost both depend on the model. + */ + SyntheticUserDriverSpec: { + /** Model Name */ + model_name: string; + model_provider: components["schemas"]["ModelProviderName"]; + }; /** TabooProperties */ TabooProperties: { /** @@ -9298,6 +9382,21 @@ export interface components { /** Taboo Examples */ taboo_examples: string; }; + /** + * TargetRunConfigSpec + * @description How to invoke the target task on each turn — same fields a manual + * UI run would use. + */ + TargetRunConfigSpec: { + /** Model Name */ + model_name: string; + model_provider: components["schemas"]["ModelProviderName"]; + /** + * Prompt Id + * @default simple_prompt_builder + */ + prompt_id: string; + }; /** * Task * @description Represents a specific task to be performed, with associated requirements and validation rules. @@ -16939,6 +17038,82 @@ export interface operations { }; }; }; + generate_cases_api_projects__project_id__tasks__task_id__multiturn_sdg_generate_cases_post: { + parameters: { + query?: never; + header?: never; + path: { + /** @description ID of the project containing the target task. */ + project_id: string; + /** @description ID of the target task. Must be a multi-turn task. */ + task_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["GenerateCasesApiInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["GenerateCasesApiOutput"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + stream_run_cases_batch_api_projects__project_id__tasks__task_id__multiturn_sdg_run_cases_batch_post: { + parameters: { + query?: never; + header?: never; + path: { + /** @description ID of the project containing the target task. */ + project_id: string; + /** @description ID of the target task. Must be a multi-turn task. */ + task_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["RunCasesBatchApiInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; api_test_access_api_git_sync_test_access_post: { parameters: { query?: never; diff --git a/libs/server/kiln_server/server.py b/libs/server/kiln_server/server.py index 5efb8e9635..92bbc71f7c 100644 --- a/libs/server/kiln_server/server.py +++ b/libs/server/kiln_server/server.py @@ -68,6 +68,10 @@ def _get_version() -> str: "name": "Synthetic Data", "description": "Generate synthetic data for evals and fine-tuning.", }, + { + "name": "Multiturn SDG", + "description": "Generate multi-turn synthetic-user conversation datasets for eval.", + }, { "name": "Fine-tuning", "description": "Create and manage fine-tuning jobs.", From edb27f0794299d2543963e072ab8a2503a3afb01 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 21:45:12 -0700 Subject: [PATCH 08/36] refactor(studio_server/synthetic_user): driver loop polish from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename total_cost -> target_total_cost on CaseCompletedEvent and BatchCompletedEvent. The runner only sees target adapter spend; the SU driver's per-turn cost isn't rolled up here. Old name was misleading in a beta where users pick the SU model. - Thread an optional save_context through run_cases_batch and wrap the leaf-tag save. Adapter writes inside adapter.invoke still bypass — a kiln_ai-side gap shared with the chat SSE pattern, documented in the runner docstring. - Add a re-run idempotency test for _tag_leaf to lock in the spec's "set-union + sort, preserves pre-existing tags" contract. - Drop the dead UNSET/None branch in client._code_or_default; the remaining one-liner has identical behavior. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../studio_server/synthetic_user/client.py | 7 +--- .../studio_server/synthetic_user/runner.py | 34 ++++++++++++++---- .../synthetic_user/test_runner.py | 36 ++++++++++++++++++- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/app/desktop/studio_server/synthetic_user/client.py b/app/desktop/studio_server/synthetic_user/client.py index c42c88187d..1cd2125251 100644 --- a/app/desktop/studio_server/synthetic_user/client.py +++ b/app/desktop/studio_server/synthetic_user/client.py @@ -34,7 +34,6 @@ ValidationError, ) from app.desktop.studio_server.api_client.kiln_ai_server_client.types import ( - UNSET, Response, ) from app.desktop.studio_server.api_client.kiln_server_client import ( @@ -162,11 +161,7 @@ def _extract_cases_or_raise(response: Response) -> list[SyntheticUserCase]: def _code_or_default(code: object, default: str) -> str: """Resolve a typed-model `code` field that may be `UNSET` or a string.""" - if isinstance(code, str) and code: - return code - if code is UNSET or code is None: - return default - return default + return code if isinstance(code, str) and code else default def _format_validation_detail(error: HTTPValidationError) -> str: diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py index fc5a31b414..1d6924516b 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -33,6 +33,7 @@ from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError +from kiln_ai.utils.git_sync_protocols import SaveContext, default_save_context from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( @@ -86,7 +87,11 @@ class CaseCompletedEvent: chain_run_ids: list[str] leaf_run_id: str total_turns: int - total_cost: float + # Sum of the target adapter's cumulative_usage.cost across this case's + # turns. Excludes the SU driver's per-turn LLM spend — SU turns run via + # invoke_returning_run_output and never persist a TaskRun, so their cost + # isn't rolled up here. Rename in mind for when SU cost gets threaded. + target_total_cost: float @dataclass(frozen=True) @@ -101,7 +106,9 @@ class BatchCompletedEvent: successful: int failed: int batch_tag: str - total_cost: float + # Sum of CaseCompletedEvent.target_total_cost across successful cases. + # Same SU-exclusion caveat applies. + target_total_cost: float BatchEvent = ( @@ -125,6 +132,7 @@ async def run_cases_batch( turns: int = DEFAULT_TURNS, concurrency: int = CONCURRENCY, batch_tag: str | None = None, + save_context: SaveContext | None = None, ) -> AsyncIterator[BatchEvent]: """Drive `cases` concurrently against `target_task`, streaming progress. @@ -133,6 +141,14 @@ async def run_cases_batch( Events from different cases interleave; ordering WITHIN a case is `turn_completed`* → `case_completed | case_failed`. + `save_context` wraps the leaf-tag save (the one write this runner + controls). The adapter's per-turn `run.save_to_file()` inside + `adapter.invoke` does NOT take a save_context — that's a kiln_ai-side + plumbing gap shared with the chat SSE pattern, not specific to this + runner. The route still uses `@no_write_lock` because wrapping the + full streaming response in one atomic_write would block all other + writes for the batch duration. + Yields: BatchStartedEvent — once, before any case runs. TurnCompletedEvent — one per assistant turn within a case. @@ -148,6 +164,7 @@ async def run_cases_batch( raise ValueError("concurrency must be >= 1") resolved_batch_tag = batch_tag or _new_batch_tag() + save_ctx: SaveContext = save_context or default_save_context semaphore = asyncio.Semaphore(concurrency) # `None` is the end-of-stream sentinel pushed when all cases finish. queue: asyncio.Queue[BatchEvent | None] = asyncio.Queue() @@ -163,6 +180,7 @@ async def _drive_one(case_index: int, case: SyntheticUserCase) -> None: turns=turns, batch_tag=resolved_batch_tag, queue=queue, + save_ctx=save_ctx, ) # Kick the cases off BEFORE the first yield so they start running @@ -181,7 +199,7 @@ async def _close_when_done() -> None: successful = 0 failed = 0 - total_cost = 0.0 + target_total_cost = 0.0 try: yield BatchStartedEvent(batch_tag=resolved_batch_tag, num_cases=len(cases)) @@ -192,7 +210,7 @@ async def _close_when_done() -> None: yield event if isinstance(event, CaseCompletedEvent): successful += 1 - total_cost += event.total_cost + target_total_cost += event.target_total_cost elif isinstance(event, CaseFailedEvent): failed += 1 @@ -200,7 +218,7 @@ async def _close_when_done() -> None: successful=successful, failed=failed, batch_tag=resolved_batch_tag, - total_cost=total_cost, + target_total_cost=target_total_cost, ) finally: # Cancel any in-flight case tasks before awaiting the closer. Without @@ -227,6 +245,7 @@ async def _drive_one_case_and_emit( turns: int, batch_tag: str, queue: asyncio.Queue[BatchEvent | None], + save_ctx: SaveContext, ) -> None: """Run drive_case for one case, translating per-turn outcomes to events on `queue` and ending with either CaseCompletedEvent or CaseFailedEvent. @@ -285,7 +304,8 @@ async def _on_turn(*, run: TaskRun, su_message: str) -> None: # so a tag-save failure (full disk, validator rejection on a # malformed batch_tag) surfaces as case_failed, not a silent drop. leaf = result.chain[-1] - _tag_leaf(leaf, batch_tag) + async with save_ctx(): + _tag_leaf(leaf, batch_tag) await queue.put( CaseCompletedEvent( @@ -295,7 +315,7 @@ async def _on_turn(*, run: TaskRun, su_message: str) -> None: ], leaf_run_id=str(leaf.id) if leaf.id is not None else "", total_turns=len(result.chain), - total_cost=_cumulative_cost(leaf), + target_total_cost=_cumulative_cost(leaf), ) ) except Exception as e: # noqa: BLE001 — beta error surface diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/app/desktop/studio_server/synthetic_user/test_runner.py index 4de3e05505..a9bad80518 100644 --- a/app/desktop/studio_server/synthetic_user/test_runner.py +++ b/app/desktop/studio_server/synthetic_user/test_runner.py @@ -223,7 +223,7 @@ async def test_three_cases_produce_full_event_stream( assert events[-1].successful == 3 assert events[-1].failed == 0 assert events[-1].batch_tag == "testbatch" - assert events[-1].total_cost == pytest.approx(0.07) + assert events[-1].target_total_cost == pytest.approx(0.07) turn_events = [e for e in events if isinstance(e, TurnCompletedEvent)] case_done = [e for e in events if isinstance(e, CaseCompletedEvent)] @@ -282,6 +282,40 @@ async def test_leaf_is_tagged_with_synthetic_user_case_and_batch_tag( leaf.save_to_file.assert_called_once() +@pytest.mark.asyncio +async def test_leaf_tagging_is_idempotent_on_pre_tagged_leaf( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """Re-tagging a leaf that already has the runner's tags + an unrelated + user tag preserves the user tag and dedupes the runner's tags. The + spec calls this out as the property that makes re-runs safe. + """ + leaf = _fake_run("leaf") + leaf.tags = [ + "synthetic_user_case", + "synthetic_user_batch:abc123", + "user_kept_this", + ] + _patch_adapter_for_task(monkeypatch, [leaf]) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + await _collect( + run_cases_batch( + cases=[_case()], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + batch_tag="abc123", + ) + ) + + assert leaf.tags.count("synthetic_user_case") == 1 + assert leaf.tags.count("synthetic_user_batch:abc123") == 1 + assert "user_kept_this" in leaf.tags + assert leaf.tags == sorted(leaf.tags) + + @pytest.mark.asyncio async def test_auto_generates_batch_tag_when_not_provided( fake_task: Mock, monkeypatch: pytest.MonkeyPatch From fbc1e5847f5fd0d77b9a8eb571ee18029ec09736 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 22:36:32 -0700 Subject: [PATCH 09/36] chore(studio_server/synthetic_user): mild review polish - Rename DEFAULT_TURNS -> MAX_TURNS_DEFAULT to match spec naming. - Name asyncio.create_task instances so debug dumps point at this code. - Pre-assert non-empty seed_prompt in drive_case (assert-loud invariant). - Document invariants on _make_target_invoker (sequential-per-case), _tag_leaf (one-writer-per-leaf), and _close_when_done (final put on cancel path goes into the void). - Drop the unreachable generic fallback in _to_http_exception; tighten the param type to the two real subclasses so the type checker enforces exhaustiveness at the call site. - Log a warning in _format_validation_detail when every item is skipped so a silent SDK shape drift surfaces. - Tests: parameterize turns<1 with negatives, lock in _event_to_payload's unregistered-event guard, and couple the auto-batch_tag test to the public regex instead of the implementation. - Stale "Phase 3" docstring scrub + f-string cosmetic. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../studio_server/multiturn_sdg_api.py | 31 +++++++---------- .../studio_server/synthetic_user/client.py | 13 +++++++ .../synthetic_user/drive_loop.py | 13 ++++--- .../studio_server/synthetic_user/runner.py | 34 ++++++++++++++++--- .../synthetic_user/test_drive_loop.py | 26 ++++++++++++-- .../synthetic_user/test_runner.py | 9 +++-- .../studio_server/test_multiturn_sdg_api.py | 19 +++++++++++ 7 files changed, 114 insertions(+), 31 deletions(-) diff --git a/app/desktop/studio_server/multiturn_sdg_api.py b/app/desktop/studio_server/multiturn_sdg_api.py index 524920ba11..24a7c5c7e6 100644 --- a/app/desktop/studio_server/multiturn_sdg_api.py +++ b/app/desktop/studio_server/multiturn_sdg_api.py @@ -56,7 +56,7 @@ ) from app.desktop.studio_server.synthetic_user.runner import ( CONCURRENCY, - DEFAULT_TURNS, + MAX_TURNS_DEFAULT, NUM_CASES_MAX, BatchCompletedEvent, BatchEvent, @@ -127,7 +127,7 @@ class RunCasesBatchApiInput(BaseModel): ), ) turns: int = Field( - default=DEFAULT_TURNS, + default=MAX_TURNS_DEFAULT, ge=1, le=20, description=( @@ -224,7 +224,9 @@ def _jsonable(obj: Any) -> Any: raise TypeError(f"{type(obj).__name__} is not JSON serializable") -def _to_http_exception(exc: Exception) -> HTTPException: +def _to_http_exception( + exc: SyntheticUserRequestError | SyntheticUserServerError, +) -> HTTPException: """Translate SyntheticUserClient's typed exceptions to HTTPExceptions. Returns the exception rather than raising so callers can `raise … from exc` @@ -245,21 +247,14 @@ def _to_http_exception(exc: Exception) -> HTTPException: status_code=status, detail={"code": exc.code, "message": exc.message}, ) - if isinstance(exc, SyntheticUserServerError): - # Preserve the upstream 5xx (502 → 502, 503 → 503, ...). Anything - # unrecognized falls to a clean 500. - status = ( - exc.status_code if exc.status_code and 500 <= exc.status_code < 600 else 500 - ) - return HTTPException( - status_code=status, - detail={"code": exc.code, "message": exc.message}, - ) - # Generic fallback — shouldn't happen for known typed exceptions but - # keeps the helper exhaustive at the call site. + # SyntheticUserServerError: preserve the upstream 5xx (502 → 502, + # 503 → 503, ...). Anything unrecognized falls to a clean 500. + status = ( + exc.status_code if exc.status_code and 500 <= exc.status_code < 600 else 500 + ) return HTTPException( - status_code=500, - detail={"code": "internal_error", "message": str(exc)}, + status_code=status, + detail={"code": exc.code, "message": exc.message}, ) @@ -343,7 +338,7 @@ async def stream_run_cases_batch( status_code=400, detail={ "code": "invalid_case_shape", - "message": (f"Could not parse cases against the SDK shape: {exc}"), + "message": f"Could not parse cases against the SDK shape: {exc}", }, ) from exc diff --git a/app/desktop/studio_server/synthetic_user/client.py b/app/desktop/studio_server/synthetic_user/client.py index 1cd2125251..a91a03ccdd 100644 --- a/app/desktop/studio_server/synthetic_user/client.py +++ b/app/desktop/studio_server/synthetic_user/client.py @@ -172,11 +172,24 @@ def _format_validation_detail(error: HTTPValidationError) -> str: if not isinstance(detail, list): return "Validation error (no detail)." parts: list[str] = [] + skipped = 0 for item in detail: if not isinstance(item, ValidationError): + skipped += 1 continue loc = ".".join(str(x) for x in item.loc) parts.append(f"{loc}: {item.msg}") if not parts: + # The SDK's HTTPValidationError.detail had items the SDK couldn't + # parse as ValidationError — a shape we don't expect today. Log + # so we can spot the discrepancy if it ever appears in the wild, + # instead of silently returning the empty fallback. + if skipped: + logger.warning( + "HTTPValidationError carried %d non-ValidationError detail item(s); " + "raw detail repr: %r", + skipped, + detail, + ) return "Validation error (no detail)." return "Validation error: " + "; ".join(parts) diff --git a/app/desktop/studio_server/synthetic_user/drive_loop.py b/app/desktop/studio_server/synthetic_user/drive_loop.py index 93dea6cd11..c44a1ca146 100644 --- a/app/desktop/studio_server/synthetic_user/drive_loop.py +++ b/app/desktop/studio_server/synthetic_user/drive_loop.py @@ -24,10 +24,10 @@ class TargetInvoker(Protocol): - """Callable that invokes the target task for one turn. Phase 3 wraps - `adapter_for_task(task, run_config).invoke` to satisfy this; tests - pass in a fake. Keeps the drive loop target-agnostic — it just cares - about the persisted TaskRun that comes back. + """Callable that invokes the target task for one turn. The runner + wraps `adapter_for_task(task, run_config).invoke` to satisfy this; + tests pass in a fake. Keeps the drive loop target-agnostic — it just + cares about the persisted TaskRun that comes back. """ async def __call__( @@ -91,6 +91,11 @@ async def drive_case( """ if turns < 1: raise ValueError(f"turns must be >= 1, got {turns}") + # Assert-loud on missing seed. An empty string would silently flow + # into the target adapter and surface as a confusing model-side error + # rather than a clean "the case is malformed" signal. + if not case.seed_prompt: + raise ValueError("case.seed_prompt must be a non-empty string") user_msg: str = case.seed_prompt prev_run: TaskRun | None = None diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py index 1d6924516b..48b9bfe02c 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -48,7 +48,7 @@ # Module constants. Sized for an MVP beta — easy to bump in one place. NUM_CASES_MAX = 10 -DEFAULT_TURNS = 5 +MAX_TURNS_DEFAULT = 5 CONCURRENCY = 4 # Tag scheme. `_TAG_SU_CASE` lets the dataset view filter to all SU-generated @@ -129,7 +129,7 @@ async def run_cases_batch( target_task: Task, target_run_config: KilnAgentRunConfigProperties, su_driver_config: SyntheticUserDriverConfig, - turns: int = DEFAULT_TURNS, + turns: int = MAX_TURNS_DEFAULT, concurrency: int = CONCURRENCY, batch_tag: str | None = None, save_context: SaveContext | None = None, @@ -186,16 +186,27 @@ async def _drive_one(case_index: int, case: SyntheticUserCase) -> None: # Kick the cases off BEFORE the first yield so they start running # concurrently with consumer setup. If the consumer disconnects between # BatchStartedEvent and the drain loop, the `finally` below still - # cancels them. - case_tasks = [asyncio.create_task(_drive_one(i, c)) for i, c in enumerate(cases)] + # cancels them. Tasks are named so asyncio debug dumps and pending- + # task warnings point at this code path. + case_tasks = [ + asyncio.create_task( + _drive_one(i, c), name=f"su_case_{i}_{resolved_batch_tag[:6]}" + ) + for i, c in enumerate(cases) + ] async def _close_when_done() -> None: # `return_exceptions=True` so a stray bug doesn't leave the queue # draining forever; we surface failures via per-case CaseFailedEvent. + # On the cancel path (consumer disconnect), the drain loop has + # already exited, so the final `put(None)` is into-the-void — + # harmless because the queue is unbounded and never re-read. await asyncio.gather(*case_tasks, return_exceptions=True) await queue.put(None) - closer = asyncio.create_task(_close_when_done()) + closer = asyncio.create_task( + _close_when_done(), name=f"su_closer_{resolved_batch_tag[:6]}" + ) successful = 0 failed = 0 @@ -351,6 +362,13 @@ def _make_target_invoker( attribution context in `input_source.properties` while subsequent runs carry only the slim `{batch_tag, turn_index}` — the case is recoverable by walking `parent_task_run_id` to the root. + + Concurrency: the returned closure is NOT safe to invoke concurrently. + `nonlocal turn_index` is incremented per call; concurrent callers + would race on the increment and the resulting `is_root` flag. + `drive_case` calls it sequentially within a single case (the + `for _ in range(turns)` loop), which is the contract; cases are + isolated by having their own closure with their own `turn_index`. """ adapter = adapter_for_task(target_task, target_run_config) turn_index = 0 @@ -443,6 +461,12 @@ def _tag_leaf(leaf: TaskRun, batch_tag: str) -> None: Tags are deduplicated (treated as a set then sorted) so re-runs against an already-tagged leaf are idempotent. A save_to_file exception surfaces to the caller (which converts to CaseFailedEvent). + + Reentrancy: the read-modify-write on `leaf.tags` assumes a single + writer per leaf. The current call shape guarantees this (each case + has its own leaf), so concurrent tagging across cases hits four + different files. A future refactor that shares leaves across cases + would need to re-introduce locking here. """ tags = set(leaf.tags or []) tags.add(_TAG_SU_CASE) diff --git a/app/desktop/studio_server/synthetic_user/test_drive_loop.py b/app/desktop/studio_server/synthetic_user/test_drive_loop.py index d4e9f2f420..bb994cad85 100644 --- a/app/desktop/studio_server/synthetic_user/test_drive_loop.py +++ b/app/desktop/studio_server/synthetic_user/test_drive_loop.py @@ -234,8 +234,12 @@ async def test_drive_case_works_without_on_turn_hook() -> None: # ───────────────────────── invariants ───────────────────────── +@pytest.mark.parametrize("bad_turns", [0, -1, -100]) @pytest.mark.asyncio -async def test_drive_case_rejects_invalid_turns() -> None: +async def test_drive_case_rejects_invalid_turns(bad_turns: int) -> None: + """Parameterized so a regression that mistyped `if turns == 0` instead of + `if turns < 1` would be caught by the negative cases. + """ invoker = _FakeInvoker(assistant_replies=[]) su = _su_driver_with_replies([]) @@ -244,7 +248,25 @@ async def test_drive_case_rejects_invalid_turns() -> None: case=_case(), target_invoker=invoker, su_driver=su, - turns=0, + turns=bad_turns, + ) + + +@pytest.mark.asyncio +async def test_drive_case_rejects_empty_seed_prompt() -> None: + """An empty seed would silently flow into the target adapter and + surface as a confusing model-side error; assert-loud at the boundary + so this fails as the unambiguous "malformed case" it is. + """ + invoker = _FakeInvoker(assistant_replies=[]) + su = _su_driver_with_replies([]) + + with pytest.raises(ValueError, match="seed_prompt"): + await drive_case( + case=_case(seed=""), + target_invoker=invoker, + su_driver=su, + turns=1, ) diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/app/desktop/studio_server/synthetic_user/test_runner.py index a9bad80518..41c4393a7c 100644 --- a/app/desktop/studio_server/synthetic_user/test_runner.py +++ b/app/desktop/studio_server/synthetic_user/test_runner.py @@ -8,6 +8,7 @@ """ import asyncio +import re from typing import Any from unittest.mock import AsyncMock, Mock @@ -333,9 +334,13 @@ async def test_auto_generates_batch_tag_when_not_provided( ) ) + # Couple the assertion to the public contract — the auto-generated + # tag must satisfy the batch_tag input regex `[A-Za-z0-9_-]{1,64}` so + # it can be passed back as `batch_tag` on a subsequent run. Avoids + # restating the implementation (uuid4().hex[:12]), which would lock + # us in to a specific length/charset just because that's what we picked. started = next(e for e in events if isinstance(e, BatchStartedEvent)) - assert len(started.batch_tag) == 12 - assert all(c in "0123456789abcdef" for c in started.batch_tag) + assert re.fullmatch(r"[A-Za-z0-9_-]{1,64}", started.batch_tag) is not None # ───────────────────────── per-case failure isolation ───────────────────────── diff --git a/app/desktop/studio_server/test_multiturn_sdg_api.py b/app/desktop/studio_server/test_multiturn_sdg_api.py index 4b8652c599..7eb4a75503 100644 --- a/app/desktop/studio_server/test_multiturn_sdg_api.py +++ b/app/desktop/studio_server/test_multiturn_sdg_api.py @@ -450,6 +450,25 @@ async def _exploding_runner(**_kwargs) -> AsyncIterator: assert "upstream catastrophe" in failed_evt["message"] +def test_event_to_payload_raises_on_unregistered_event_type() -> None: + """If a new BatchEvent dataclass is added but not registered in + `_EVENT_NAMES`, `_event_to_payload` must fail loud at test time, not + silently emit a malformed SSE frame in production. Locks in the + defensive RuntimeError so a contributor adding a new event without + updating the map fails the build instead of shipping a wire bug. + """ + from dataclasses import dataclass + + from app.desktop.studio_server.multiturn_sdg_api import _event_to_payload + + @dataclass(frozen=True) + class _UnregisteredEvent: + x: int = 1 + + with pytest.raises(RuntimeError, match="Unregistered BatchEvent type"): + _event_to_payload(_UnregisteredEvent()) # type: ignore[arg-type] + + def test_run_cases_batch_jsonable_typeerror_surfaces_as_batch_failed( client: TestClient, patch_task_from_id, patch_api_key ) -> None: From 14304e76d3105b28842c78ccb017f8960f160ed0 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 22:48:50 -0700 Subject: [PATCH 10/36] refactor(studio_server/synthetic_user): decompose root input_source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root TaskRun's input_source.properties now carries the decomposed SU case context — persona, goal, behavior_guidance (when present), seed_prompt — instead of the opaque tagged blob. Lets dataset readers and eval tooling inspect SU attribution by direct property access rather than re-parsing the XML each time. The blob is losslessly reconstructable from these fields via build_synthetic_user_info if a downstream tool needs the original wire form. Parse happens once per case in _build_input_source on the root turn; the SU driver constructor already validated the blob, so the re-parse here can't surface a new error class. behavior_guidance is omitted when the parser returns None (the DataSource validator rejects empty strings). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../studio_server/synthetic_user/runner.py | 26 +++++++-- .../synthetic_user/test_runner.py | 53 +++++++++++++++++-- 2 files changed, 70 insertions(+), 9 deletions(-) diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py index 48b9bfe02c..3fe8c7ca03 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -32,7 +32,10 @@ from kiln_ai.datamodel.task_run import TaskRun from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig -from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError +from kiln_ai.synthetic_user.parser import ( + SyntheticUserInfoParseError, + parse_synthetic_user_info, +) from kiln_ai.utils.git_sync_protocols import SaveContext, default_save_context from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam @@ -413,9 +416,15 @@ def _build_input_source( extra property keys alongside the required `model_name` / `model_provider` / `adapter_name`. - Root run carries the full opaque blob + seed_prompt so anyone landing - on this run can recover the case context without walking the chain. - Subsequent runs carry only the slim batch_tag/turn_index pair. + Root run carries the decomposed case context (persona / goal / + behavior_guidance / seed_prompt) so a reader landing on the run can + inspect the SU configuration without re-parsing the blob. Subsequent + runs carry only the slim batch_tag/turn_index pair; the full context + is recoverable by walking `parent_task_run_id` to the root. + + The blob itself is not persisted — it's losslessly reconstructable + from the decomposed fields via `build_synthetic_user_info` if any + downstream tool needs the original wire form. """ props: dict[str, str | int | float] = { "model_name": su_driver_config.model_name, @@ -425,7 +434,14 @@ def _build_input_source( "turn_index": turn_index, } if is_root: - props["synthetic_user_info"] = case.synthetic_user_info + # Parse is cheap (regex on a short string) and was already + # validated when the SU driver was built — re-parsing here can't + # surface a new error class. + info = parse_synthetic_user_info(case.synthetic_user_info) + props["persona"] = info.persona + props["goal"] = info.goal + if info.behavior_guidance: + props["behavior_guidance"] = info.behavior_guidance props["seed_prompt"] = case.seed_prompt # The validator rejects empty string property values. Strip any that diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/app/desktop/studio_server/synthetic_user/test_runner.py index 41c4393a7c..712e2cc979 100644 --- a/app/desktop/studio_server/synthetic_user/test_runner.py +++ b/app/desktop/studio_server/synthetic_user/test_runner.py @@ -490,11 +490,13 @@ async def slow_invoke(**_kwargs: Any) -> Mock: @pytest.mark.asyncio -async def test_root_input_source_carries_blob_and_seed_prompt( +async def test_root_input_source_carries_decomposed_case_context( fake_task: Mock, monkeypatch: pytest.MonkeyPatch ) -> None: - """First adapter.invoke call gets an input_source with the full SU - case context: the opaque blob + seed_prompt. + """First adapter.invoke call gets an input_source with the decomposed + SU case context (persona/goal/behavior_guidance/seed_prompt) rather + than the opaque blob. Lets dataset readers inspect attribution + without re-parsing. """ captured: list[dict[str, Any]] = [] @@ -524,10 +526,53 @@ async def _capture(**kwargs: Any) -> Mock: assert props["model_provider"] == "openrouter" assert props["batch_tag"] == "rb1" assert props["turn_index"] == 1 - assert props["synthetic_user_info"] == case.synthetic_user_info + # Decomposed — no opaque blob persisted. + assert "synthetic_user_info" not in props + assert props["persona"] == "persona-0" + assert props["goal"] == "goal-0" + assert props["behavior_guidance"] == "guidance-0" assert props["seed_prompt"] == case.seed_prompt +@pytest.mark.asyncio +async def test_root_input_source_omits_behavior_guidance_when_absent( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """`behavior_guidance` is the only optional SU info field — when the + blob omits the tag (or has empty content), the property must be + absent rather than empty-stringed. The DataSource validator rejects + empty strings, so the filter in `_build_input_source` would otherwise + save us here, but we want the wire shape predictable either way. + """ + captured: list[dict[str, Any]] = [] + + async def _capture(**kwargs: Any) -> Mock: + captured.append(kwargs) + return _fake_run(f"r-{len(captured)}") + + _patch_adapter_for_task(monkeypatch, _capture) + _patch_su_driver(monkeypatch, replies_per_case=["x"]) + + case_no_guidance = SyntheticUserCase( + seed_prompt="hello", + synthetic_user_info="pg", + ) + await _collect( + run_cases_batch( + cases=[case_no_guidance], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=1, + ) + ) + + props = captured[0]["input_source"].properties + assert props["persona"] == "p" + assert props["goal"] == "g" + assert "behavior_guidance" not in props + + @pytest.mark.asyncio async def test_non_root_input_source_is_slim( fake_task: Mock, monkeypatch: pytest.MonkeyPatch From bdd7c4b569a1aa585cb9799cdee846c17ef82378 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:09:01 -0700 Subject: [PATCH 11/36] refactor(synthetic_user): thread SU driver cost into total_cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SyntheticUserDriver.respond now returns (message, cost) — the per-call cost is read from the in-memory TaskRun's usage.cost (the only place SU spend surfaces, since SU turns aren't persisted as TaskRuns). drive_case accumulates su_total_cost across turns and exposes it on DriveCaseResult. The runner adds it to the leaf's cumulative_usage.cost to produce an honest CaseCompletedEvent.total_cost — renamed from target_total_cost since the field now reports total spend, not just the target adapter's. BatchCompletedEvent.total_cost sums across successful cases the same way. Matters now because the SU model is user-selectable: someone picking Sonnet for higher-quality probes would have had ~half their spend invisible under the old target-only total. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../synthetic_user/drive_loop.py | 12 ++++- .../studio_server/synthetic_user/runner.py | 23 ++++----- .../synthetic_user/test_drive_loop.py | 9 ++-- .../synthetic_user/test_runner.py | 50 +++++++++++++++++-- .../studio_server/test_multiturn_sdg_api.py | 10 ++-- libs/core/kiln_ai/synthetic_user/driver.py | 24 +++++++-- .../kiln_ai/synthetic_user/test_driver.py | 45 +++++++++++++++-- 7 files changed, 138 insertions(+), 35 deletions(-) diff --git a/app/desktop/studio_server/synthetic_user/drive_loop.py b/app/desktop/studio_server/synthetic_user/drive_loop.py index c44a1ca146..725c0a23b5 100644 --- a/app/desktop/studio_server/synthetic_user/drive_loop.py +++ b/app/desktop/studio_server/synthetic_user/drive_loop.py @@ -58,9 +58,15 @@ class DriveCaseResult: `chain` is the list of persisted TaskRuns the adapter produced (leaf last). There is no stop_reason field — every case ends after exactly `turns` iterations by design. + + `su_total_cost` sums the SU driver's per-turn LLM cost across the + case. SU turns aren't persisted as TaskRuns, so this is the only + place that spend surfaces — the runner adds it to the target's + `cumulative_usage.cost` to produce an honest total. """ chain: list[TaskRun] + su_total_cost: float async def drive_case( @@ -101,6 +107,7 @@ async def drive_case( prev_run: TaskRun | None = None prev_trace: list[ChatCompletionMessageParam] | None = None chain: list[TaskRun] = [] + su_total_cost: float = 0.0 for _ in range(turns): new_run = await target_invoker( @@ -112,7 +119,8 @@ async def drive_case( # The SU driver does the role filtering / role swap / invariant # checks itself. We pass the new run's cumulative trace as-is. - su_message = await su_driver.respond(new_run.trace or []) + su_message, su_cost = await su_driver.respond(new_run.trace or []) + su_total_cost += su_cost if on_turn is not None: await on_turn(run=new_run, su_message=su_message) @@ -121,4 +129,4 @@ async def drive_case( prev_run = new_run prev_trace = new_run.trace - return DriveCaseResult(chain=chain) + return DriveCaseResult(chain=chain, su_total_cost=su_total_cost) diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py index 3fe8c7ca03..9570f12e61 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -90,11 +90,11 @@ class CaseCompletedEvent: chain_run_ids: list[str] leaf_run_id: str total_turns: int - # Sum of the target adapter's cumulative_usage.cost across this case's - # turns. Excludes the SU driver's per-turn LLM spend — SU turns run via - # invoke_returning_run_output and never persist a TaskRun, so their cost - # isn't rolled up here. Rename in mind for when SU cost gets threaded. - target_total_cost: float + # Honest total spend for this case: target adapter's + # cumulative_usage.cost on the leaf + the SU driver's per-turn cost + # summed across turns. SU turns aren't persisted as TaskRuns; drive_case + # threads their cost through DriveCaseResult.su_total_cost. + total_cost: float @dataclass(frozen=True) @@ -109,9 +109,8 @@ class BatchCompletedEvent: successful: int failed: int batch_tag: str - # Sum of CaseCompletedEvent.target_total_cost across successful cases. - # Same SU-exclusion caveat applies. - target_total_cost: float + # Sum of CaseCompletedEvent.total_cost across successful cases. + total_cost: float BatchEvent = ( @@ -213,7 +212,7 @@ async def _close_when_done() -> None: successful = 0 failed = 0 - target_total_cost = 0.0 + total_cost = 0.0 try: yield BatchStartedEvent(batch_tag=resolved_batch_tag, num_cases=len(cases)) @@ -224,7 +223,7 @@ async def _close_when_done() -> None: yield event if isinstance(event, CaseCompletedEvent): successful += 1 - target_total_cost += event.target_total_cost + total_cost += event.total_cost elif isinstance(event, CaseFailedEvent): failed += 1 @@ -232,7 +231,7 @@ async def _close_when_done() -> None: successful=successful, failed=failed, batch_tag=resolved_batch_tag, - target_total_cost=target_total_cost, + total_cost=total_cost, ) finally: # Cancel any in-flight case tasks before awaiting the closer. Without @@ -329,7 +328,7 @@ async def _on_turn(*, run: TaskRun, su_message: str) -> None: ], leaf_run_id=str(leaf.id) if leaf.id is not None else "", total_turns=len(result.chain), - target_total_cost=_cumulative_cost(leaf), + total_cost=_cumulative_cost(leaf) + result.su_total_cost, ) ) except Exception as e: # noqa: BLE001 — beta error surface diff --git a/app/desktop/studio_server/synthetic_user/test_drive_loop.py b/app/desktop/studio_server/synthetic_user/test_drive_loop.py index bb994cad85..4083f37206 100644 --- a/app/desktop/studio_server/synthetic_user/test_drive_loop.py +++ b/app/desktop/studio_server/synthetic_user/test_drive_loop.py @@ -91,10 +91,13 @@ async def __call__( return _fake_run(new_trace, run_id=f"run-turn-{len(self.calls)}") -def _su_driver_with_replies(replies: list[str]) -> Mock: - """Mock(spec=SyntheticUserDriver) with respond() returning canned strings.""" +def _su_driver_with_replies(replies: list[str], cost_per_reply: float = 0.0) -> Mock: + """Mock(spec=SyntheticUserDriver) with respond() returning canned + (message, cost) tuples. `cost_per_reply` lets cost-aware tests inject + a non-zero per-call cost; defaults to 0.0 for the legacy tests. + """ drv = Mock(spec=SyntheticUserDriver) - drv.respond = AsyncMock(side_effect=replies) + drv.respond = AsyncMock(side_effect=[(r, cost_per_reply) for r in replies]) return drv diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/app/desktop/studio_server/synthetic_user/test_runner.py index 712e2cc979..b400e0c849 100644 --- a/app/desktop/studio_server/synthetic_user/test_runner.py +++ b/app/desktop/studio_server/synthetic_user/test_runner.py @@ -122,7 +122,9 @@ def _ctor(blob, config): # noqa: ARG001 else list(replies_per_case) ) instance = Mock(spec=SyntheticUserDriver) - instance.respond = AsyncMock(side_effect=replies) + # respond() returns (message, cost). Tests that don't care about + # cost get 0.0 — the runner adds it to total_cost regardless. + instance.respond = AsyncMock(side_effect=[(r, 0.0) for r in replies]) return instance monkeypatch.setattr(runner_mod, "SyntheticUserDriver", _ctor) @@ -224,7 +226,7 @@ async def test_three_cases_produce_full_event_stream( assert events[-1].successful == 3 assert events[-1].failed == 0 assert events[-1].batch_tag == "testbatch" - assert events[-1].target_total_cost == pytest.approx(0.07) + assert events[-1].total_cost == pytest.approx(0.07) turn_events = [e for e in events if isinstance(e, TurnCompletedEvent)] case_done = [e for e in events if isinstance(e, CaseCompletedEvent)] @@ -235,6 +237,48 @@ async def test_three_cases_produce_full_event_stream( assert ev.total_turns == 1 +@pytest.mark.asyncio +async def test_total_cost_sums_target_and_su_driver_spend( + fake_task: Mock, monkeypatch: pytest.MonkeyPatch +) -> None: + """CaseCompletedEvent.total_cost = leaf.cumulative_usage.cost + sum of + SU driver's per-turn cost across the case. BatchCompletedEvent.total_cost + sums those across successful cases. Locks in the honest-totals contract. + """ + leaf_a = _fake_run("a-leaf", cost=0.10) + leaf_b = _fake_run("b-leaf", cost=0.05) + _patch_adapter_for_task( + monkeypatch, + [_fake_run("a-1"), leaf_a, _fake_run("b-1"), leaf_b], + ) + + # Each case's driver gets two replies at $0.01 each → $0.02 SU per case. + def _ctor(blob, config): # noqa: ARG001 + instance = Mock(spec=SyntheticUserDriver) + instance.respond = AsyncMock(side_effect=[("u2", 0.01), ("u3", 0.01)]) + return instance + + monkeypatch.setattr(runner_mod, "SyntheticUserDriver", _ctor) + + events = await _collect( + run_cases_batch( + cases=[_case(0), _case(1)], + target_task=fake_task, + target_run_config=_target_run_config(), + su_driver_config=_su_driver_config(), + turns=2, + concurrency=1, + ) + ) + + case_a, case_b = (e for e in events if isinstance(e, CaseCompletedEvent)) + assert case_a.total_cost == pytest.approx(0.10 + 0.02) + assert case_b.total_cost == pytest.approx(0.05 + 0.02) + + batch = next(e for e in events if isinstance(e, BatchCompletedEvent)) + assert batch.total_cost == pytest.approx(0.12 + 0.07) + + @pytest.mark.asyncio async def test_turn_completed_event_carries_su_message_and_trace( fake_task: Mock, monkeypatch: pytest.MonkeyPatch @@ -361,7 +405,7 @@ def _ctor(blob, config): # noqa: ARG001 if idx == 1: raise SyntheticUserInfoParseError("bad blob") instance = Mock(spec=SyntheticUserDriver) - instance.respond = AsyncMock(return_value="ok") + instance.respond = AsyncMock(return_value=("ok", 0.0)) return instance _patch_su_driver_factory(monkeypatch, _ctor) diff --git a/app/desktop/studio_server/test_multiturn_sdg_api.py b/app/desktop/studio_server/test_multiturn_sdg_api.py index 7eb4a75503..b582746d0a 100644 --- a/app/desktop/studio_server/test_multiturn_sdg_api.py +++ b/app/desktop/studio_server/test_multiturn_sdg_api.py @@ -309,7 +309,7 @@ def test_run_cases_batch_emits_full_sse_event_stream( chain_run_ids=["r0a"], leaf_run_id="r0a", total_turns=1, - target_total_cost=0.01, + total_cost=0.01, ), CaseFailedEvent( case_index=1, @@ -317,7 +317,7 @@ def test_run_cases_batch_emits_full_sse_event_stream( message="missing required tag", ), BatchCompletedEvent( - successful=1, failed=1, batch_tag="testbatch", target_total_cost=0.01 + successful=1, failed=1, batch_tag="testbatch", total_cost=0.01 ), ] @@ -365,7 +365,7 @@ async def _fake_runner(**_kwargs) -> AsyncIterator: "successful": 1, "failed": 1, "batch_tag": "testbatch", - "target_total_cost": 0.01, + "total_cost": 0.01, } @@ -391,9 +391,7 @@ def test_run_cases_batch_jsonable_handles_message_usage_in_trace( {"role": "assistant", "content": "hi back", "usage": usage}, # type: ignore[typeddict-unknown-key] ], ), - BatchCompletedEvent( - successful=1, failed=0, batch_tag="tb", target_total_cost=0.001 - ), + BatchCompletedEvent(successful=1, failed=0, batch_tag="tb", total_cost=0.001), ] async def _fake_runner(**_kwargs) -> AsyncIterator: diff --git a/libs/core/kiln_ai/synthetic_user/driver.py b/libs/core/kiln_ai/synthetic_user/driver.py index dee2625d79..aec3028404 100644 --- a/libs/core/kiln_ai/synthetic_user/driver.py +++ b/libs/core/kiln_ai/synthetic_user/driver.py @@ -76,12 +76,19 @@ def __init__( ) self._adapter = adapter_for_task(self._task, self._run_config) - async def respond(self, conversation: list[ChatCompletionMessageParam]) -> str: - """Return the SU's next message. + async def respond( + self, conversation: list[ChatCompletionMessageParam] + ) -> tuple[str, float]: + """Return the SU's next message and the cost of producing it. `conversation` is in the eval frame and must end on an `assistant` (target) turn — the SU is responding to that turn. Drive-loop termination is the caller's concern; this just produces one reply. + + The cost is read from the in-memory TaskRun's `usage.cost`. SU + turns aren't persisted, so this is the only place the cost + surfaces — callers (e.g. drive_case) must thread it forward if + they want to report total spend. """ # 1) Filter to visible roles (drop system/tool if present). visible = [ @@ -118,11 +125,20 @@ async def respond(self, conversation: list[ChatCompletionMessageParam]) -> str: # 4) Adapter call. invoke_returning_run_output returns # (TaskRun, RunOutput) without writing the TaskRun to disk. - _task_run, run_output = await self._adapter.invoke_returning_run_output( + task_run, run_output = await self._adapter.invoke_returning_run_output( user_input, prior_trace=prior_trace ) raw = run_output.output if not isinstance(raw, str): raise RuntimeError("synthetic user returned non-string output") - return raw + # Per-call cost from the in-memory TaskRun. None when the + # provider doesn't surface pricing (rare; defaults to 0.0 so the + # downstream sum stays well-defined). + cost = ( + float(task_run.usage.cost) + if task_run.usage is not None and task_run.usage.cost is not None + else 0.0 + ) + + return raw, cost diff --git a/libs/core/kiln_ai/synthetic_user/test_driver.py b/libs/core/kiln_ai/synthetic_user/test_driver.py index 754ebcceb9..50ba574e24 100644 --- a/libs/core/kiln_ai/synthetic_user/test_driver.py +++ b/libs/core/kiln_ai/synthetic_user/test_driver.py @@ -13,6 +13,7 @@ from kiln_ai.adapters.run_output import RunOutput from kiln_ai.datamodel.datamodel_enums import ModelProviderName from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.datamodel.usage import Usage from kiln_ai.synthetic_user import driver as driver_mod from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig @@ -35,14 +36,27 @@ def _fake_run_output(text: str | dict = "hi from the SU") -> RunOutput: return RunOutput(output=text, intermediate_outputs=None) -def _patch_adapter(monkeypatch: pytest.MonkeyPatch, return_value: RunOutput) -> Mock: +def _patch_adapter( + monkeypatch: pytest.MonkeyPatch, + return_value: RunOutput, + cost: float | None = None, +) -> Mock: """Replace adapter_for_task with a stub returning a mock adapter whose invoke_returning_run_output yields (Mock(spec=TaskRun), return_value). Returns the mock adapter so tests can assert call args. + + Pass `cost` to populate the in-memory TaskRun's `.usage.cost`; when + omitted, `.usage` is None and `respond()` should report cost=0.0. """ + task_run = Mock(spec=TaskRun) + task_run.usage = ( + Usage(input_tokens=0, output_tokens=0, total_tokens=0, cost=cost) + if cost is not None + else None + ) adapter = Mock() adapter.invoke_returning_run_output = AsyncMock( - return_value=(Mock(spec=TaskRun), return_value) + return_value=(task_run, return_value) ) monkeypatch.setattr( driver_mod, "adapter_for_task", lambda task, run_config: adapter @@ -92,9 +106,12 @@ def test_construction_raises_on_empty_required_tag( @pytest.mark.asyncio -async def test_respond_returns_adapter_output( +async def test_respond_returns_adapter_output_and_zero_cost_when_unset( monkeypatch: pytest.MonkeyPatch, ) -> None: + """When the provider doesn't surface pricing, cost defaults to 0.0 + so downstream sums stay well-defined. + """ adapter = _patch_adapter(monkeypatch, _fake_run_output("the SU's reply")) drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) conversation: list[ChatCompletionMessageParam] = [ @@ -102,12 +119,30 @@ async def test_respond_returns_adapter_output( {"role": "assistant", "content": "a1"}, ] - out = await drv.respond(conversation) + message, cost = await drv.respond(conversation) - assert out == "the SU's reply" + assert message == "the SU's reply" + assert cost == 0.0 adapter.invoke_returning_run_output.assert_awaited_once() +@pytest.mark.asyncio +async def test_respond_returns_cost_from_task_run_usage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Per-call cost is read from the in-memory TaskRun's `usage.cost`.""" + _patch_adapter(monkeypatch, _fake_run_output("hi"), cost=0.0123) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + ] + + _, cost = await drv.respond(conversation) + + assert cost == pytest.approx(0.0123) + + @pytest.mark.asyncio async def test_respond_role_swaps_and_prepends_system_prompt( monkeypatch: pytest.MonkeyPatch, From c3cd3214a66eb38ef349b851bf335f88f5eb120a Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:14:59 -0700 Subject: [PATCH 12/36] chore(synthetic_user): drop dead empty-string filter in _build_input_source Every input to the filter has stronger upstream protection now: seed_prompt is asserted non-empty in drive_case; persona and goal are required-non-empty by parse_synthetic_user_info; behavior_guidance is already conditionally skipped if None; the remaining keys are Pydantic- validated or non-string. The filter was guarding nothing. The DataSource validator stays as the real backstop. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/desktop/studio_server/synthetic_user/runner.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/app/desktop/studio_server/synthetic_user/runner.py index 9570f12e61..ae5f5a8229 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/app/desktop/studio_server/synthetic_user/runner.py @@ -443,11 +443,7 @@ def _build_input_source( props["behavior_guidance"] = info.behavior_guidance props["seed_prompt"] = case.seed_prompt - # The validator rejects empty string property values. Strip any that - # would trip it (shouldn't happen for required fields in practice, but - # this protects against an empty seed_prompt or a malformed config). - filtered = {k: v for k, v in props.items() if not (isinstance(v, str) and v == "")} - return DataSource(type=DataSourceType.synthetic, properties=filtered) + return DataSource(type=DataSourceType.synthetic, properties=props) # ───────────────────────── small utilities ───────────────────────── From 54ee96f26f97d3edd6e6af2ec09b015b6f70ee10 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Tue, 2 Jun 2026 00:24:08 -0700 Subject: [PATCH 13/36] refactor(synthetic_user): move runner + drive_loop into the kiln-ai SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure relocation + boundary update; behavior unchanged. run_cases_batch and drive_case now live at libs/core/kiln_ai/synthetic_user/{runner,drive_loop}.py alongside the existing SyntheticUserDriver. Same neighborhood as EvalRunner / RagJobRunner / ExtractorRunner — runners belong in libs/core. To make libs/core SDK-agnostic, introduce a small kiln_ai.synthetic_user.SyntheticUserCase Pydantic model (two fields, field-identical to the kiln_server SDK's case shape). The multiturn_sdg_api route validates dicts straight into the libs/core type via Pydantic, so the runner never sees the SDK class. The SDK case is still used for `/generate_cases` output via `to_dict()` — nothing about that pro-gated authoring path changes. Tests move with the code. studio_server keeps only the SDK-wrapper SyntheticUserClient and the FastAPI route, which is exactly the established shape for eval_api driving EvalRunner. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../studio_server/multiturn_sdg_api.py | 61 ++++++++++--------- .../studio_server/test_multiturn_sdg_api.py | 2 +- app/web_ui/src/lib/api_schema.d.ts | 4 +- libs/core/kiln_ai/synthetic_user/__init__.py | 3 + libs/core/kiln_ai/synthetic_user/case.py | 25 ++++++++ .../kiln_ai}/synthetic_user/drive_loop.py | 5 +- .../core/kiln_ai}/synthetic_user/runner.py | 12 +--- libs/core/kiln_ai/synthetic_user/test_case.py | 27 ++++++++ .../synthetic_user/test_drive_loop.py | 10 +-- .../kiln_ai}/synthetic_user/test_runner.py | 16 ++--- 10 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 libs/core/kiln_ai/synthetic_user/case.py rename {app/desktop/studio_server => libs/core/kiln_ai}/synthetic_user/drive_loop.py (97%) rename {app/desktop/studio_server => libs/core/kiln_ai}/synthetic_user/runner.py (98%) create mode 100644 libs/core/kiln_ai/synthetic_user/test_case.py rename {app/desktop/studio_server => libs/core/kiln_ai}/synthetic_user/test_drive_loop.py (98%) rename {app/desktop/studio_server => libs/core/kiln_ai}/synthetic_user/test_runner.py (98%) diff --git a/app/desktop/studio_server/multiturn_sdg_api.py b/app/desktop/studio_server/multiturn_sdg_api.py index 24a7c5c7e6..24809a5387 100644 --- a/app/desktop/studio_server/multiturn_sdg_api.py +++ b/app/desktop/studio_server/multiturn_sdg_api.py @@ -39,22 +39,9 @@ from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig from kiln_ai.datamodel.task import Task from kiln_ai.datamodel.usage import MessageUsage +from kiln_ai.synthetic_user.case import SyntheticUserCase as RunnerCase from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig -from kiln_server.cancellable_streaming_response import CancellableStreamingResponse -from kiln_server.git_sync_decorators import build_save_context, no_write_lock -from kiln_server.task_api import task_from_id -from kiln_server.utils.agent_checks.policy import agent_policy_require_approval -from pydantic import BaseModel, Field - -from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( - SyntheticUserCase, -) -from app.desktop.studio_server.synthetic_user.client import ( - SyntheticUserClient, - SyntheticUserRequestError, - SyntheticUserServerError, -) -from app.desktop.studio_server.synthetic_user.runner import ( +from kiln_ai.synthetic_user.runner import ( CONCURRENCY, MAX_TURNS_DEFAULT, NUM_CASES_MAX, @@ -66,6 +53,17 @@ TurnCompletedEvent, run_cases_batch, ) +from kiln_server.cancellable_streaming_response import CancellableStreamingResponse +from kiln_server.git_sync_decorators import build_save_context, no_write_lock +from kiln_server.task_api import task_from_id +from kiln_server.utils.agent_checks.policy import agent_policy_require_approval +from pydantic import BaseModel, Field + +from app.desktop.studio_server.synthetic_user.client import ( + SyntheticUserClient, + SyntheticUserRequestError, + SyntheticUserServerError, +) from app.desktop.studio_server.utils.copilot_utils import get_copilot_api_key logger = logging.getLogger(__name__) @@ -73,16 +71,16 @@ # ───────────────────────── Pydantic API models ───────────────────────── -# Cases ride the wire as `list[dict[str, Any]]` rather than a Pydantic -# mirror — the SDK's SyntheticUserCase is the single source of truth, and -# we round-trip via `to_dict` / `from_dict` at the boundary. Trade-off: -# TS bindings type cases as `Record` instead of getting -# per-field autocomplete. +# Cases ride the wire as `list[dict[str, Any]]`: the kiln_server SDK +# emits cases as attrs models with `to_dict()` (used by `/generate_cases` +# below) and the libs/core runner consumes `SyntheticUserCase` (Pydantic). +# Both are field-identical; this route validates dicts straight into the +# libs/core type via Pydantic. Trade-off: TS bindings type cases as +# `Record` instead of getting per-field autocomplete. SyntheticUserCaseDict = dict[str, Any] _CASE_DICT_DESCRIPTION = ( - "A SyntheticUserCase as returned by kiln_server's /generate. Shape: " - "{seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info " - "value is an XML-tagged blob: " + "A SyntheticUserCase. Shape: {seed_prompt: str, synthetic_user_info: str}. " + "The synthetic_user_info value is an XML-tagged blob: " ".......... " "Parsed client-side by kiln_ai.synthetic_user.parser." ) @@ -328,17 +326,20 @@ async def stream_run_cases_batch( task = task_from_id(project_id, task_id) _guard_multiturn(task) - # SDK from_dict raises on missing/wrong-typed required fields; - # surface that as a clean 400 instead of letting it explode inside - # the SSE generator. + # Parse dict → libs/core RunnerCase. Pydantic raises ValidationError + # on missing keys or empty strings; surface as a clean 400 instead + # of letting it explode inside the SSE generator. We go straight to + # the libs/core type (skipping the SDK round-trip) because the two + # shapes are field-identical and the runner only needs the libs/core + # one. try: - sdk_cases = [SyntheticUserCase.from_dict(c) for c in input.cases] - except (KeyError, TypeError, ValueError) as exc: + runner_cases = [RunnerCase.model_validate(c) for c in input.cases] + except Exception as exc: # noqa: BLE001 — Pydantic ValidationError + any future shape drift raise HTTPException( status_code=400, detail={ "code": "invalid_case_shape", - "message": f"Could not parse cases against the SDK shape: {exc}", + "message": f"Could not parse cases against the runner shape: {exc}", }, ) from exc @@ -349,7 +350,7 @@ async def stream_run_cases_batch( async def event_generator(): try: async for event in run_cases_batch( - cases=sdk_cases, + cases=runner_cases, target_task=task, target_run_config=target_run_config, su_driver_config=su_driver_config, diff --git a/app/desktop/studio_server/test_multiturn_sdg_api.py b/app/desktop/studio_server/test_multiturn_sdg_api.py index b582746d0a..4e264a5cb1 100644 --- a/app/desktop/studio_server/test_multiturn_sdg_api.py +++ b/app/desktop/studio_server/test_multiturn_sdg_api.py @@ -29,7 +29,7 @@ SyntheticUserRequestError, SyntheticUserServerError, ) -from app.desktop.studio_server.synthetic_user.runner import ( +from kiln_ai.synthetic_user.runner import ( BatchCompletedEvent, BatchStartedEvent, CaseCompletedEvent, diff --git a/app/web_ui/src/lib/api_schema.d.ts b/app/web_ui/src/lib/api_schema.d.ts index 6d1d7a7850..f05dd26d78 100644 --- a/app/web_ui/src/lib/api_schema.d.ts +++ b/app/web_ui/src/lib/api_schema.d.ts @@ -6626,7 +6626,7 @@ export interface components { GenerateCasesApiOutput: { /** * Cases - * @description A SyntheticUserCase as returned by kiln_server's /generate. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. + * @description A SyntheticUserCase. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. */ cases: { [key: string]: unknown; @@ -8575,7 +8575,7 @@ export interface components { RunCasesBatchApiInput: { /** * Cases - * @description Cases as returned by /generate_cases, optionally edited. A SyntheticUserCase as returned by kiln_server's /generate. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. + * @description Cases as returned by /generate_cases, optionally edited. A SyntheticUserCase. Shape: {seed_prompt: str, synthetic_user_info: str}. The synthetic_user_info value is an XML-tagged blob: .......... Parsed client-side by kiln_ai.synthetic_user.parser. */ cases: { [key: string]: unknown; diff --git a/libs/core/kiln_ai/synthetic_user/__init__.py b/libs/core/kiln_ai/synthetic_user/__init__.py index 7676c14f60..72c8ee36b5 100644 --- a/libs/core/kiln_ai/synthetic_user/__init__.py +++ b/libs/core/kiln_ai/synthetic_user/__init__.py @@ -7,11 +7,13 @@ - `SyntheticUserDriver` — construct once per case, call `respond()` per turn. - `SyntheticUserInfo` / `SyntheticUserDriverConfig` — typed configs. +- `SyntheticUserCase` — input contract for the multi-turn drive loop. - `parse_synthetic_user_info` / `build_synthetic_user_info` — tagged-blob codec. - `SyntheticUserInfoParseError` — raised on malformed blob. - `role_swap` — exposed for callers that drive the loop themselves. """ +from kiln_ai.synthetic_user.case import SyntheticUserCase from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import ( SyntheticUserDriverConfig, @@ -26,6 +28,7 @@ from kiln_ai.synthetic_user.role_swap import role_swap __all__ = [ + "SyntheticUserCase", "SyntheticUserDriver", "SyntheticUserDriverConfig", "SyntheticUserInfo", diff --git a/libs/core/kiln_ai/synthetic_user/case.py b/libs/core/kiln_ai/synthetic_user/case.py new file mode 100644 index 0000000000..c88567c462 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/case.py @@ -0,0 +1,25 @@ +"""SyntheticUserCase — input contract for the multi-turn SU runner. + +Two-field shape: `seed_prompt` (the opening user message) and +`synthetic_user_info` (an opaque tagged blob the SyntheticUserDriver +parses on construction). + +Field-identical to the kiln_server SDK's `SyntheticUserCase`. Lives here +in libs/core so the runner has no dependency on the vendored SDK in +`app/desktop/`; studio_server's FastAPI route converts SDK case → this +case at the wire boundary. +""" + +from pydantic import BaseModel, Field + + +class SyntheticUserCase(BaseModel): + """One case for the multi-turn SU drive loop. + + `seed_prompt` is the first user-side message sent into the target + task. `synthetic_user_info` is the persona/goal/behavior_guidance + blob the driver parses to build the SU's system prompt. + """ + + seed_prompt: str = Field(..., min_length=1) + synthetic_user_info: str = Field(..., min_length=1) diff --git a/app/desktop/studio_server/synthetic_user/drive_loop.py b/libs/core/kiln_ai/synthetic_user/drive_loop.py similarity index 97% rename from app/desktop/studio_server/synthetic_user/drive_loop.py rename to libs/core/kiln_ai/synthetic_user/drive_loop.py index 725c0a23b5..1587ec8538 100644 --- a/app/desktop/studio_server/synthetic_user/drive_loop.py +++ b/libs/core/kiln_ai/synthetic_user/drive_loop.py @@ -15,13 +15,10 @@ from typing import Protocol from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.case import SyntheticUserCase from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam -from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( - SyntheticUserCase, -) - class TargetInvoker(Protocol): """Callable that invokes the target task for one turn. The runner diff --git a/app/desktop/studio_server/synthetic_user/runner.py b/libs/core/kiln_ai/synthetic_user/runner.py similarity index 98% rename from app/desktop/studio_server/synthetic_user/runner.py rename to libs/core/kiln_ai/synthetic_user/runner.py index ae5f5a8229..b8456c039b 100644 --- a/app/desktop/studio_server/synthetic_user/runner.py +++ b/libs/core/kiln_ai/synthetic_user/runner.py @@ -30,6 +30,8 @@ from kiln_ai.datamodel.task import Task from kiln_ai.datamodel.task_output import DataSource, DataSourceType from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.case import SyntheticUserCase +from kiln_ai.synthetic_user.drive_loop import TargetInvoker, drive_case from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig from kiln_ai.synthetic_user.parser import ( @@ -39,14 +41,6 @@ from kiln_ai.utils.git_sync_protocols import SaveContext, default_save_context from kiln_ai.utils.open_ai_types import ChatCompletionMessageParam -from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( - SyntheticUserCase, -) -from app.desktop.studio_server.synthetic_user.drive_loop import ( - TargetInvoker, - drive_case, -) - logger = logging.getLogger(__name__) # Module constants. Sized for an MVP beta — easy to bump in one place. @@ -331,7 +325,7 @@ async def _on_turn(*, run: TaskRun, su_message: str) -> None: total_cost=_cumulative_cost(leaf) + result.su_total_cost, ) ) - except Exception as e: # noqa: BLE001 — beta error surface + except Exception as e: # Adapter network errors, model misconfig, save_to_file blow-up, # anything unexpected. Log with full traceback; emit a structured # failure so the batch invariant holds (every case gets one event). diff --git a/libs/core/kiln_ai/synthetic_user/test_case.py b/libs/core/kiln_ai/synthetic_user/test_case.py new file mode 100644 index 0000000000..6c315bce89 --- /dev/null +++ b/libs/core/kiln_ai/synthetic_user/test_case.py @@ -0,0 +1,27 @@ +"""Unit tests for SyntheticUserCase.""" + +import pytest +from pydantic import ValidationError + +from kiln_ai.synthetic_user.case import SyntheticUserCase + + +def test_case_accepts_valid_fields() -> None: + case = SyntheticUserCase( + seed_prompt="hi there", + synthetic_user_info="pg", + ) + assert case.seed_prompt == "hi there" + assert "persona>p None: + with pytest.raises(ValidationError): + SyntheticUserCase( + seed_prompt="", synthetic_user_info="pg" + ) + + +def test_case_rejects_empty_synthetic_user_info() -> None: + with pytest.raises(ValidationError): + SyntheticUserCase(seed_prompt="hi", synthetic_user_info="") diff --git a/app/desktop/studio_server/synthetic_user/test_drive_loop.py b/libs/core/kiln_ai/synthetic_user/test_drive_loop.py similarity index 98% rename from app/desktop/studio_server/synthetic_user/test_drive_loop.py rename to libs/core/kiln_ai/synthetic_user/test_drive_loop.py index 4083f37206..18fb99b1e0 100644 --- a/app/desktop/studio_server/synthetic_user/test_drive_loop.py +++ b/libs/core/kiln_ai/synthetic_user/test_drive_loop.py @@ -11,16 +11,10 @@ import pytest from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user.case import SyntheticUserCase +from kiln_ai.synthetic_user.drive_loop import DriveCaseResult, drive_case from kiln_ai.synthetic_user.driver import SyntheticUserDriver -from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( - SyntheticUserCase, -) -from app.desktop.studio_server.synthetic_user.drive_loop import ( - DriveCaseResult, - drive_case, -) - # ───────────────────────── helpers / fixtures ───────────────────────── diff --git a/app/desktop/studio_server/synthetic_user/test_runner.py b/libs/core/kiln_ai/synthetic_user/test_runner.py similarity index 98% rename from app/desktop/studio_server/synthetic_user/test_runner.py rename to libs/core/kiln_ai/synthetic_user/test_runner.py index b400e0c849..49a37ed666 100644 --- a/app/desktop/studio_server/synthetic_user/test_runner.py +++ b/libs/core/kiln_ai/synthetic_user/test_runner.py @@ -21,15 +21,12 @@ from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties, ToolsRunConfig from kiln_ai.datamodel.task import Task from kiln_ai.datamodel.task_run import TaskRun +from kiln_ai.synthetic_user import runner as runner_mod +from kiln_ai.synthetic_user.case import SyntheticUserCase from kiln_ai.synthetic_user.driver import SyntheticUserDriver from kiln_ai.synthetic_user.models import SyntheticUserDriverConfig from kiln_ai.synthetic_user.parser import SyntheticUserInfoParseError - -from app.desktop.studio_server.api_client.kiln_ai_server_client.models import ( - SyntheticUserCase, -) -from app.desktop.studio_server.synthetic_user import runner as runner_mod -from app.desktop.studio_server.synthetic_user.runner import ( +from kiln_ai.synthetic_user.runner import ( BatchCompletedEvent, BatchEvent, BatchStartedEvent, @@ -39,7 +36,6 @@ run_cases_batch, ) - # ───────────────────────── helpers / fixtures ───────────────────────── @@ -113,7 +109,7 @@ def _patch_su_driver( """ call_counter = {"i": 0} - def _ctor(blob, config): # noqa: ARG001 + def _ctor(blob, config): idx = call_counter["i"] call_counter["i"] += 1 replies = ( @@ -253,7 +249,7 @@ async def test_total_cost_sums_target_and_su_driver_spend( ) # Each case's driver gets two replies at $0.01 each → $0.02 SU per case. - def _ctor(blob, config): # noqa: ARG001 + def _ctor(blob, config): instance = Mock(spec=SyntheticUserDriver) instance.respond = AsyncMock(side_effect=[("u2", 0.01), ("u3", 0.01)]) return instance @@ -399,7 +395,7 @@ async def test_malformed_blob_surfaces_as_case_failed( """ call_counter = {"i": 0} - def _ctor(blob, config): # noqa: ARG001 + def _ctor(blob, config): idx = call_counter["i"] call_counter["i"] += 1 if idx == 1: From ec1bb72bd079fb3fbf065f77ef4596e77a85809a Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:04:55 -0700 Subject: [PATCH 14/36] fix(synthetic_user): filter tool-dispatch-only assistant turns before role_swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool-using targets emit assistant turns with content=None and tool_calls set — pure tool dispatches, not user-facing speech. Pre-this-fix, those hit role_swap's strict-content invariant and crashed the SU run. Gemini's suggestion (coerce None → "") would have let them through but degraded the SU LLM's conversation view to consecutive user turns with empty content — silently worse than the crash. The right place to filter is at the driver, next to the existing visible_message_roles filter — "what's visible to the SU" is the driver's responsibility. role_swap stays strict on None content (the trip wire for any caller bypassing the driver's filter). Filter predicate: drop assistant turns where content is None. Keep assistant turns that carry text alongside tool_calls — the text is user-facing speech the SU should respond to. Addresses gemini-code-assist comment on PR #1441 / role_swap.py without applying the suggested empty-string coercion. Co-Authored-By: Claude Opus 4.7 (1M context) --- libs/core/kiln_ai/synthetic_user/driver.py | 22 +++++- .../kiln_ai/synthetic_user/test_driver.py | 77 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/libs/core/kiln_ai/synthetic_user/driver.py b/libs/core/kiln_ai/synthetic_user/driver.py index aec3028404..5d54e605d3 100644 --- a/libs/core/kiln_ai/synthetic_user/driver.py +++ b/libs/core/kiln_ai/synthetic_user/driver.py @@ -31,6 +31,16 @@ ) +def _is_tool_dispatch_only(msg: ChatCompletionMessageParam) -> bool: + """True if `msg` is an assistant turn with no user-facing text (i.e., + a pure tool-call dispatch). The SU LLM shouldn't see these — they're + actions the target took, not speech the SU is reacting to. Assistant + turns that carry text alongside tool_calls are NOT filtered out + (the text is the user-facing part). + """ + return msg["role"] == "assistant" and msg.get("content") is None + + class SyntheticUserDriver: """Plays one synthetic user across multiple turns. @@ -96,7 +106,15 @@ async def respond( for m in conversation if m["role"] in self._driver_config.visible_message_roles ] - # 2) Invariants this driver enforces (moved from kiln_server's + # 2) Drop tool-dispatch-only assistant turns (content=None). These + # represent the target calling a tool with no user-facing text — + # not speech the SU should "see" or respond to. Assistant turns + # that carry BOTH text and tool_calls are kept, since the text is + # user-facing. role_swap stays strict on None content (the trip + # wire); upstream filtering is what keeps it from firing on a + # tool-using target. + visible = [m for m in visible if not _is_tool_dispatch_only(m)] + # 3) Invariants this driver enforces (moved from kiln_server's # removed /respond route validator). if not visible: raise ValueError("No LLM-visible messages in conversation.") @@ -106,7 +124,7 @@ async def respond( "is responding to that turn." ) - # 3) Role-swap then assemble prior_trace + input. The last swapped + # 4) Role-swap then assemble prior_trace + input. The last swapped # turn becomes the LLM `input`; everything before it goes into # `prior_trace` with a system message prepended. swapped = role_swap(visible) diff --git a/libs/core/kiln_ai/synthetic_user/test_driver.py b/libs/core/kiln_ai/synthetic_user/test_driver.py index 50ba574e24..2892482364 100644 --- a/libs/core/kiln_ai/synthetic_user/test_driver.py +++ b/libs/core/kiln_ai/synthetic_user/test_driver.py @@ -204,6 +204,83 @@ async def test_respond_filters_visible_message_roles( assert prior_trace[1] == {"role": "assistant", "content": "u1"} +@pytest.mark.asyncio +async def test_respond_drops_tool_dispatch_only_assistant_turns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A tool-using target produces assistant turns with content=None and + tool_calls set. Those are actions the target took, not speech the SU + should respond to. They must be filtered out before role_swap so the + swap's strict-content invariant doesn't fire and so the SU's view of + the conversation stays a coherent text-only dialog. + """ + adapter = _patch_adapter(monkeypatch, _fake_run_output("ok")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + # Shape that comes out of a tool-using target: user → tool-dispatch + # assistant (content=None) → final text-response assistant. + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "what's the weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, # type: ignore[typeddict-item] + {"role": "assistant", "content": "It's 72F."}, + ] + + await drv.respond(conversation) + + call = adapter.invoke_returning_run_output.await_args + # The tool-dispatch turn (None content) was dropped. After role-swap: + # eval [user, assistant("It's 72F.")] → LLM [assistant, user("It's 72F.")] + # Last swapped turn becomes `input`; prior_trace is [sys, ...rest]. + assert call.args[0] == "It's 72F." + prior_trace = call.kwargs["prior_trace"] + assert len(prior_trace) == 2 # [sys, swapped user 'what's the weather'] + assert prior_trace[0]["content"] == drv._system_prompt + assert prior_trace[1] == {"role": "assistant", "content": "what's the weather?"} + + +@pytest.mark.asyncio +async def test_respond_keeps_assistant_turns_with_text_and_tool_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Some providers emit assistant turns with BOTH text content AND + tool_calls in the same message ("Let me look that up." + dispatch). + The text is user-facing — the SU should see it. Only None-content + turns get dropped. + """ + adapter = _patch_adapter(monkeypatch, _fake_run_output("ok")) + drv = SyntheticUserDriver(_BLOB, _DRIVER_CONFIG) + conversation: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "what's the weather?"}, + { + "role": "assistant", + "content": "Let me look that up.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, # type: ignore[typeddict-item] + ] + + await drv.respond(conversation) + + call = adapter.invoke_returning_run_output.await_args + # The combined text+tool_calls assistant turn is NOT filtered out — + # its text becomes the input the SU responds to. + assert call.args[0] == "Let me look that up." + + @pytest.mark.asyncio async def test_respond_with_custom_visible_roles( monkeypatch: pytest.MonkeyPatch, From edf3794363329d31539becd206bbe033eba22e77 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Wed, 3 Jun 2026 07:53:29 -0700 Subject: [PATCH 15/36] =?UTF-8?q?chore(synthetic=5Fuser):=20trim=20comment?= =?UTF-8?q?s=20=E2=80=94=20fix=20inaccuracies,=20strip=20stale=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix comment numbering in driver.py (4→5), correct "greedy" to "non-greedy" in parser.py, remove inaccurate drive-loop claim from studio_server __init__. Strip historical /respond migration references, remove app-layer concerns (SSE, @no_write_lock) from SDK-level docstrings, deduplicate cost-attribution explanations across driver/runner/drive_loop. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../studio_server/synthetic_user/__init__.py | 7 +- .../studio_server/synthetic_user/client.py | 11 +--- libs/core/kiln_ai/synthetic_user/driver.py | 40 ++++-------- libs/core/kiln_ai/synthetic_user/models.py | 3 +- libs/core/kiln_ai/synthetic_user/parser.py | 2 +- libs/core/kiln_ai/synthetic_user/runner.py | 65 ++++--------------- 6 files changed, 30 insertions(+), 98 deletions(-) diff --git a/app/desktop/studio_server/synthetic_user/__init__.py b/app/desktop/studio_server/synthetic_user/__init__.py index 72b84653f6..3808145613 100644 --- a/app/desktop/studio_server/synthetic_user/__init__.py +++ b/app/desktop/studio_server/synthetic_user/__init__.py @@ -1,10 +1,5 @@ """Studio_server-side wrapper for the kiln_server synthetic-user `/generate` -endpoint, plus the multi-turn synthetic-data-generation drive loop. - -Per-turn synthetic-user invocation itself lives in -`libs/core/kiln_ai/synthetic_user/` (calls the LLM with the user's keys); -this module covers only the authoring HTTP call and the runner that -orchestrates target invocation + SU response across multiple cases. +endpoint. Exports the client and its typed exception hierarchy. """ from app.desktop.studio_server.synthetic_user.client import ( diff --git a/app/desktop/studio_server/synthetic_user/client.py b/app/desktop/studio_server/synthetic_user/client.py index a91a03ccdd..8e1750b5c0 100644 --- a/app/desktop/studio_server/synthetic_user/client.py +++ b/app/desktop/studio_server/synthetic_user/client.py @@ -5,14 +5,9 @@ 1. Error classification. The SDK parses 200/401/422/500/502 into typed models; we translate those into the wrapper's typed exception hierarchy so callers never inspect raw HTTP status codes. -2. No retry. `/generate` is a once-per-batch authoring call (not a per-turn - hot loop), and kiln_server's own pipeline already retries transient - provider failures once before returning 502. A 502 reaching us is a - genuine per-batch failure that should surface, not a transient to retry. - -No `/respond` here. Per-turn synthetic-user invocation lives in -`libs/core/kiln_ai/synthetic_user/`. This wrapper exists only for the -authoring call. +2. No retry. `/generate` is a once-per-batch authoring call, and + kiln_server already retries transient provider failures internally + before returning 502. A 502 reaching us is a genuine failure. """ import logging diff --git a/libs/core/kiln_ai/synthetic_user/driver.py b/libs/core/kiln_ai/synthetic_user/driver.py index 5d54e605d3..0f8858ffe8 100644 --- a/libs/core/kiln_ai/synthetic_user/driver.py +++ b/libs/core/kiln_ai/synthetic_user/driver.py @@ -1,5 +1,4 @@ -"""Per-turn synthetic-user driver — the OSS-side replacement for kiln_server's -removed `/respond` route. +"""Per-turn synthetic-user driver. Wraps a kiln_ai LiteLLM adapter and exposes a single async `respond()` that: 1. Filters the eval-frame conversation to `visible_message_roles`. @@ -7,11 +6,8 @@ 3. Calls the adapter with the persona system prompt prepended as `prior_trace` and the latest swapped user turn as `input`. -The driver does NOT persist `TaskRun`s — it uses -`adapter.invoke_returning_run_output(...)` which builds an in-memory run -without writing to disk. The eval-dataset chain consists only of *target* -TaskRuns (created elsewhere by `adapter.invoke(...)`). The SU is an -orchestration component, not an eval-dataset row. +The driver does NOT persist `TaskRun`s — it builds in-memory runs only. +The eval-dataset chain consists only of *target* TaskRuns. """ from kiln_ai.adapters.adapter_registry import adapter_for_task @@ -89,16 +85,10 @@ def __init__( async def respond( self, conversation: list[ChatCompletionMessageParam] ) -> tuple[str, float]: - """Return the SU's next message and the cost of producing it. + """Return the SU's next message and the per-call cost. `conversation` is in the eval frame and must end on an `assistant` - (target) turn — the SU is responding to that turn. Drive-loop - termination is the caller's concern; this just produces one reply. - - The cost is read from the in-memory TaskRun's `usage.cost`. SU - turns aren't persisted, so this is the only place the cost - surfaces — callers (e.g. drive_case) must thread it forward if - they want to report total spend. + (target) turn. Drive-loop termination is the caller's concern. """ # 1) Filter to visible roles (drop system/tool if present). visible = [ @@ -106,16 +96,10 @@ async def respond( for m in conversation if m["role"] in self._driver_config.visible_message_roles ] - # 2) Drop tool-dispatch-only assistant turns (content=None). These - # represent the target calling a tool with no user-facing text — - # not speech the SU should "see" or respond to. Assistant turns - # that carry BOTH text and tool_calls are kept, since the text is - # user-facing. role_swap stays strict on None content (the trip - # wire); upstream filtering is what keeps it from firing on a - # tool-using target. + # 2) Drop tool-dispatch-only assistant turns (content=None). + # See _is_tool_dispatch_only for rationale. visible = [m for m in visible if not _is_tool_dispatch_only(m)] - # 3) Invariants this driver enforces (moved from kiln_server's - # removed /respond route validator). + # 3) Invariants. if not visible: raise ValueError("No LLM-visible messages in conversation.") if visible[-1]["role"] != "assistant": @@ -141,8 +125,7 @@ async def respond( } prior_trace: list[ChatCompletionMessageParam] = [system_msg, *swapped[:-1]] - # 4) Adapter call. invoke_returning_run_output returns - # (TaskRun, RunOutput) without writing the TaskRun to disk. + # 5) Adapter call (in-memory, no disk write). task_run, run_output = await self._adapter.invoke_returning_run_output( user_input, prior_trace=prior_trace ) @@ -150,9 +133,8 @@ async def respond( if not isinstance(raw, str): raise RuntimeError("synthetic user returned non-string output") - # Per-call cost from the in-memory TaskRun. None when the - # provider doesn't surface pricing (rare; defaults to 0.0 so the - # downstream sum stays well-defined). + # Per-call cost; defaults to 0.0 when the provider doesn't + # surface pricing. cost = ( float(task_run.usage.cost) if task_run.usage is not None and task_run.usage.cost is not None diff --git a/libs/core/kiln_ai/synthetic_user/models.py b/libs/core/kiln_ai/synthetic_user/models.py index e532a7fe3b..e67339960a 100644 --- a/libs/core/kiln_ai/synthetic_user/models.py +++ b/libs/core/kiln_ai/synthetic_user/models.py @@ -3,8 +3,7 @@ `SyntheticUserInfo` is the parsed form of the tagged blob the server sends on `/generate`; the parser produces it and `prompt.render_system_prompt` consumes it. `SyntheticUserDriverConfig` carries the per-eval runtime -config — model, provider, role visibility — moved here from the kiln_server -wire (where it used to live on the now-deleted `/respond` request). +config — model, provider, role visibility. """ from typing import Literal diff --git a/libs/core/kiln_ai/synthetic_user/parser.py b/libs/core/kiln_ai/synthetic_user/parser.py index 0d40c7b4d4..bee8db464c 100644 --- a/libs/core/kiln_ai/synthetic_user/parser.py +++ b/libs/core/kiln_ai/synthetic_user/parser.py @@ -22,7 +22,7 @@ class SyntheticUserInfoParseError(ValueError): def _extract(blob: str, tag: str) -> str | None: - """Greedy first-match. Returns the trimmed content, or None if no match.""" + """First match, non-greedy content. Returns trimmed content, or None.""" m = re.search(rf"<{tag}>(.*?)", blob, re.DOTALL) if m is None: return None diff --git a/libs/core/kiln_ai/synthetic_user/runner.py b/libs/core/kiln_ai/synthetic_user/runner.py index b8456c039b..84168e4f46 100644 --- a/libs/core/kiln_ai/synthetic_user/runner.py +++ b/libs/core/kiln_ai/synthetic_user/runner.py @@ -1,21 +1,8 @@ """Batch runner — fans drive_case out across N cases, streams BatchEvents. -`run_cases_batch` is an async generator yielding typed events so the -upcoming SSE endpoint can map each to a `data:` frame without -re-instrumenting. Cases run concurrently under an asyncio.Semaphore. - -Responsibilities owned here (not in drive_case): -- Building the per-case `SyntheticUserDriver` (catches malformed - `synthetic_user_info` blob → `CaseFailedEvent` for that case only). -- Building the per-case `TargetInvoker` that wraps `adapter.invoke` with - the SU-attribution `input_source` — opaque blob on the root run, slim - `{batch_tag, turn_index}` on subsequent turns. -- Tagging the leaf TaskRun for downstream eval-dataset discovery. -- Per-case error isolation (typed driver / loop / unexpected exceptions - → `CaseFailedEvent` without affecting other in-flight cases). -- Bounded cleanup on consumer disconnect — case tasks are cancelled - before the closer awaits them, so a browser disconnect doesn't keep - the request alive for the duration of every in-flight case. +`run_cases_batch` is an async generator yielding typed events. Cases run +concurrently under an asyncio.Semaphore. Per-case failures surface as +`CaseFailedEvent` without affecting other in-flight cases. """ import asyncio @@ -43,7 +30,7 @@ logger = logging.getLogger(__name__) -# Module constants. Sized for an MVP beta — easy to bump in one place. +# Module constants. NUM_CASES_MAX = 10 MAX_TURNS_DEFAULT = 5 CONCURRENCY = 4 @@ -84,10 +71,7 @@ class CaseCompletedEvent: chain_run_ids: list[str] leaf_run_id: str total_turns: int - # Honest total spend for this case: target adapter's - # cumulative_usage.cost on the leaf + the SU driver's per-turn cost - # summed across turns. SU turns aren't persisted as TaskRuns; drive_case - # threads their cost through DriveCaseResult.su_total_cost. + # Target adapter cost + SU driver cost for this case. total_cost: float @@ -137,14 +121,6 @@ async def run_cases_batch( Events from different cases interleave; ordering WITHIN a case is `turn_completed`* → `case_completed | case_failed`. - `save_context` wraps the leaf-tag save (the one write this runner - controls). The adapter's per-turn `run.save_to_file()` inside - `adapter.invoke` does NOT take a save_context — that's a kiln_ai-side - plumbing gap shared with the chat SSE pattern, not specific to this - runner. The route still uses `@no_write_lock` because wrapping the - full streaming response in one atomic_write would block all other - writes for the batch duration. - Yields: BatchStartedEvent — once, before any case runs. TurnCompletedEvent — one per assistant turn within a case. @@ -254,18 +230,13 @@ async def _drive_one_case_and_emit( queue: asyncio.Queue[BatchEvent | None], save_ctx: SaveContext, ) -> None: - """Run drive_case for one case, translating per-turn outcomes to events - on `queue` and ending with either CaseCompletedEvent or CaseFailedEvent. - - All upstream work — SyntheticUserDriver construction, target_invoker - construction, drive_case execution, leaf tagging — runs inside a single - try/except so any failure surfaces as a CaseFailedEvent rather than - escaping into `asyncio.gather(return_exceptions=True)` (which would - silently drop the case from the event stream). + """Run drive_case for one case, emitting events on `queue`. + + Everything runs inside a single try/except so any failure surfaces as + a CaseFailedEvent rather than silently dropping the case. """ try: - # Construct the per-case SU driver. Parses the blob immediately; - # a malformed blob fails this case without affecting others. + # Malformed blob fails this case without affecting others. try: su_driver = SyntheticUserDriver(case.synthetic_user_info, su_driver_config) except SyntheticUserInfoParseError as e: @@ -404,20 +375,10 @@ def _build_input_source( ) -> DataSource: """Attribute the user-side input on this turn to the SU driver model. - Reuses `DataSourceType.synthetic` (a model produced this text) rather - than inventing a new type — the existing validator accepts arbitrary - extra property keys alongside the required `model_name` / - `model_provider` / `adapter_name`. - Root run carries the decomposed case context (persona / goal / - behavior_guidance / seed_prompt) so a reader landing on the run can - inspect the SU configuration without re-parsing the blob. Subsequent - runs carry only the slim batch_tag/turn_index pair; the full context - is recoverable by walking `parent_task_run_id` to the root. - - The blob itself is not persisted — it's losslessly reconstructable - from the decomposed fields via `build_synthetic_user_info` if any - downstream tool needs the original wire form. + behavior_guidance / seed_prompt). Subsequent runs carry only the slim + batch_tag/turn_index pair — full context is recoverable by walking + parent_task_run_id to the root. """ props: dict[str, str | int | float] = { "model_name": su_driver_config.model_name, From d032dcf1853dabd7cd5f300107ec0791f4261c17 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Wed, 3 Jun 2026 08:35:12 -0700 Subject: [PATCH 16/36] chore(web_ui): strip zero-width space from multiturn_composer comment Stray U+200B (zero-width space) between "disables/" and "spinners" in a comment tripped eslint no-irregular-whitespace. Likely a paste artifact from Leonard's recent commit; fixed in passing during the merge. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/web_ui/src/lib/ui/conversation/multiturn_composer.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/web_ui/src/lib/ui/conversation/multiturn_composer.svelte b/app/web_ui/src/lib/ui/conversation/multiturn_composer.svelte index 57b394792f..37465778a8 100644 --- a/app/web_ui/src/lib/ui/conversation/multiturn_composer.svelte +++ b/app/web_ui/src/lib/ui/conversation/multiturn_composer.svelte @@ -194,7 +194,7 @@ collapse via an empty-conditional slot without regressing append-mode layout (full-width button vs. fork-mode's right-aligned button next to Cancel). The cancel-button markup is the only difference. While - submitting, FormContainer disables/​spinners the Send button and we + submitting, FormContainer disables/spinners the Send button and we disable the textarea — the area stays put rather than being hidden. --> {#if mode === "fork" && on_cancel} Date: Thu, 4 Jun 2026 13:02:53 -0700 Subject: [PATCH 17/36] feat(copilot): multi-turn save endpoint + classify-spec stub Extends spec_with_copilot to handle multi-turn synthetic-user batches. When the request carries a `multi_turn.batch_tag`, the endpoint: - finds existing chain leaves tagged synthetic_user_batch: - applies the spec's eval + golden filter tags to them - creates Eval with evaluation_data_type=full_trace and train_set_filter_id=None - skips example synthesis and TaskRun creation A request-shape validator enforces mutual exclusion with sdg_session_config and requires evaluate_full_trace=True for the multi-turn path. Also adds classify_spec_description as a stub endpoint that returns 501 until the kiln_server classifier ships. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/desktop/studio_server/copilot_api.py | 267 +++++++++++++----- app/desktop/studio_server/test_copilot_api.py | 237 +++++++++++++++- .../studio_server/utils/copilot_utils.py | 44 ++- app/web_ui/src/lib/api_schema.d.ts | 142 +++++++++- 4 files changed, 611 insertions(+), 79 deletions(-) diff --git a/app/desktop/studio_server/copilot_api.py b/app/desktop/studio_server/copilot_api.py index 32aa20d359..78b4982d28 100644 --- a/app/desktop/studio_server/copilot_api.py +++ b/app/desktop/studio_server/copilot_api.py @@ -44,8 +44,10 @@ ) from app.desktop.studio_server.utils.copilot_utils import ( create_dataset_task_runs, + find_multi_turn_chain_leaves, generate_copilot_examples, get_copilot_api_key, + tag_multi_turn_chains_for_eval, ) from app.desktop.studio_server.utils.response_utils import unwrap_response from fastapi import FastAPI, HTTPException, Path @@ -60,7 +62,7 @@ SyntheticDataGenerationStepConfig, TaskSample, ) -from kiln_ai.datamodel.spec_properties import SpecProperties +from kiln_ai.datamodel.spec_properties import SpecProperties, SpecType from kiln_ai.utils.name_generator import generate_memorable_name from kiln_server.task_api import task_from_id from kiln_server.utils.spec_utils import ( @@ -76,25 +78,78 @@ SubmitAnswersRequest, ) from kiln_server.utils.agent_checks.policy import agent_policy_require_approval -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator +from typing_extensions import Self logger = logging.getLogger(__name__) +class ClassifySpecDescriptionInput(BaseModel): + """Free-text description of an eval the user wants to build. The + endpoint maps it to one of the 9 spec types and pre-fills the + property_values for that type so the v2 builder can skip the + template-carousel step entirely. + """ + + description: str = Field( + description="Free-text description of what the eval should check." + ) + task_prompt: str | None = Field( + default=None, + description="Optional task prompt for context (improves classification " + "accuracy when the spec relates to a specific task).", + ) + + +class ClassifySpecDescriptionOutput(BaseModel): + """Classified spec type + suggested name + spec_type-specific property + values. The `property_values` dict keys match the FieldConfig keys + defined for the chosen spec_type (see + app/web_ui/.../select_template/spec_templates.ts). + """ + + spec_type: SpecType = Field(description="The classified spec type.") + suggested_name: str = Field( + description="A filename-safe name for the new spec, derived from the description." + ) + property_values: dict[str, str] = Field( + description="Pre-filled property values for the chosen spec_type. " + "Keys correspond to the field_configs of that spec_type." + ) + + +class MultiTurnSaveInfo(BaseModel): + """Identifies an existing multi-turn synthetic-user batch to turn into an Eval. + The endpoint walks chains tagged with this batch_tag and applies eval/golden + filter tags instead of generating new examples. + """ + + batch_tag: str = Field( + description="The batch_tag emitted by the multi-turn synthetic-user runner " + "(see kiln_ai.synthetic_user.runner). Identifies the set of conversation " + "chains already persisted to disk that this Eval should evaluate." + ) + + class CreateSpecWithCopilotRequest(BaseModel): """Request model for creating a spec with Kiln Copilot. - This endpoint uses Kiln Copilot to: - - Generate batch examples for eval, train, and golden datasets - - Create a judge eval config - - Create an eval with appropriate template/output scores - - Create and save the spec + Two synthesis paths are supported, exactly one must be set per request: + + - **Single-turn:** caller supplies `sdg_session_config`. Endpoint calls + `generate_copilot_examples` for fresh I/O pairs, splits them into + eval/train/golden datasets, and tags new TaskRuns. - If you don't want to use copilot, use the regular POST /spec endpoint instead. + - **Multi-turn:** caller supplies `multi_turn` with a `batch_tag` pointing + at chains already on disk (created earlier by the synthetic-user runner). + Endpoint tags the existing chain leaves with eval/golden filter tags; + no new TaskRuns are created. `evaluate_full_trace` must be True. + + If you don't want copilot at all, use POST /spec instead. The client is responsible for building: - - definition: The spec definition string (use buildSpecDefinition on client) - - properties: The spec properties object (filtered, with spec_type included) + - definition: the spec definition string (buildSpecDefinition on client) + - properties: the spec properties object (filtered, with spec_type included) """ name: FilenameString @@ -108,13 +163,53 @@ class CreateSpecWithCopilotRequest(BaseModel): evaluate_full_trace: bool = False reviewed_examples: list[ReviewedExample] = Field(default_factory=list) judge_info: SyntheticDataGenerationStepConfigApi - sdg_session_config: SyntheticDataGenerationSessionConfigApi + sdg_session_config: SyntheticDataGenerationSessionConfigApi | None = None + multi_turn: MultiTurnSaveInfo | None = None task_description: str = "" task_prompt_with_example: str = "" task_sample: TaskSample | None = None + @model_validator(mode="after") + def validate_synthesis_path(self) -> Self: + if self.multi_turn is not None and self.sdg_session_config is not None: + raise ValueError( + "Pass exactly one of `multi_turn` or `sdg_session_config` — not both." + ) + if self.multi_turn is None and self.sdg_session_config is None: + raise ValueError( + "Must pass one of `multi_turn` (for multi-turn chains already on " + "disk) or `sdg_session_config` (for fresh single-turn synthesis)." + ) + if self.multi_turn is not None and not self.evaluate_full_trace: + raise ValueError( + "Multi-turn save requires `evaluate_full_trace=True` — the eval " + "evaluates full conversation traces, not single I/O pairs." + ) + return self + def connect_copilot_api(app: FastAPI): + @app.post( + "/api/copilot/classify_spec_description", + tags=["Copilot"], + openapi_extra=agent_policy_require_approval( + "Classify a free-text spec description?" + ), + ) + async def classify_spec_description( + input: ClassifySpecDescriptionInput, + ) -> ClassifySpecDescriptionOutput: + """Classify a free-text spec description into one of the 9 spec types. + + Not implemented yet — the underlying kiln_server classifier hasn't + shipped. Returns 501 so callers can detect this and fall back to + manual spec-type selection. + """ + raise HTTPException( + status_code=501, + detail="Spec classification isn't implemented yet.", + ) + @app.post( "/api/copilot/clarify_spec", tags=["Copilot"], @@ -315,10 +410,27 @@ async def create_spec_with_copilot( spec_type, request.evaluate_full_trace ) + # Multi-turn path: find existing chain leaves up front so we 404 before + # creating any models if the batch_tag matches nothing. + multi_turn_leaves: list[TaskRun] = [] + if request.multi_turn is not None: + multi_turn_leaves = find_multi_turn_chain_leaves( + task, request.multi_turn.batch_tag + ) + if not multi_turn_leaves: + raise HTTPException( + status_code=404, + detail=( + f"No multi-turn chains found for batch_tag " + f"'{request.multi_turn.batch_tag}'." + ), + ) + # Build models but don't save yet, collect all models first models_to_save: list[Eval | EvalConfig | TaskRun | Spec] = [] - # 1. Create the Eval + # 1. Create the Eval. Multi-turn has no train set (small datasets, + # not used for fine-tuning) so train_set_filter_id is None. eval = Eval( parent=task, name=request.name, @@ -326,14 +438,16 @@ async def create_spec_with_copilot( template=template, output_scores=output_scores, eval_set_filter_id=eval_set_filter_id, - train_set_filter_id=train_set_filter_id, + train_set_filter_id=( + None if request.multi_turn is not None else train_set_filter_id + ), eval_configs_filter_id=eval_configs_filter_id, template_properties=None, evaluation_data_type=evaluation_data_type, ) models_to_save.append(eval) - # 2. Create judge eval config + # 2. Create judge eval config (same in both paths) eval_config = EvalConfig( parent=eval, name=generate_memorable_name(), @@ -350,44 +464,68 @@ async def create_spec_with_copilot( # Set as default config after ID is assigned eval.current_config_id = eval_config.id - # 3. Generate examples via copilot API - api_key = get_copilot_api_key() - task_input_schema = ( - str(task.input_json_schema) if task.input_json_schema else "" - ) - task_output_schema = ( - str(task.output_json_schema) if task.output_json_schema else "" - ) - all_examples = await generate_copilot_examples( - api_key=api_key, - target_task_info=TaskInfoApi( - task_prompt=request.task_prompt_with_example, - task_input_schema=task_input_schema, - task_output_schema=task_output_schema, - ), - sdg_session_config=request.sdg_session_config, - spec_definition=request.definition, - ) - - # 4. Create TaskRuns for eval, train, and golden datasets - dataset_runs = create_dataset_task_runs( - all_examples=all_examples, - reviewed_examples=request.reviewed_examples, - eval_tag=eval_tag, - train_tag=train_tag, - golden_tag=golden_tag, - spec_name=request.name, - ) - task_runs = dataset_runs.task_runs - for run in task_runs: - run.parent = task - models_to_save.extend(task_runs) + # 3+4. Single-turn: synthesise examples + create TaskRuns. + # Multi-turn: skipped — chains already exist on disk. + task_runs: list[TaskRun] = [] + dataset_runs = None + sdg_session_config_for_spec: SyntheticDataGenerationSessionConfig | None = None + if request.multi_turn is None: + assert request.sdg_session_config is not None # validator guarantees + api_key = get_copilot_api_key() + task_input_schema = ( + str(task.input_json_schema) if task.input_json_schema else "" + ) + task_output_schema = ( + str(task.output_json_schema) if task.output_json_schema else "" + ) + all_examples = await generate_copilot_examples( + api_key=api_key, + target_task_info=TaskInfoApi( + task_prompt=request.task_prompt_with_example, + task_input_schema=task_input_schema, + task_output_schema=task_output_schema, + ), + sdg_session_config=request.sdg_session_config, + spec_definition=request.definition, + ) - # 5. Create the Spec using pre-computed definition and properties from client - topic_generation_config = request.sdg_session_config.topic_generation_config - input_generation_config = request.sdg_session_config.input_generation_config - output_generation_config = request.sdg_session_config.output_generation_config + dataset_runs = create_dataset_task_runs( + all_examples=all_examples, + reviewed_examples=request.reviewed_examples, + eval_tag=eval_tag, + train_tag=train_tag, + golden_tag=golden_tag, + spec_name=request.name, + ) + task_runs = dataset_runs.task_runs + for run in task_runs: + run.parent = task + models_to_save.extend(task_runs) + + # Snapshot the generation config on the Spec (single-turn only). + topic_cfg = request.sdg_session_config.topic_generation_config + input_cfg = request.sdg_session_config.input_generation_config + output_cfg = request.sdg_session_config.output_generation_config + sdg_session_config_for_spec = SyntheticDataGenerationSessionConfig( + topic_generation_config=SyntheticDataGenerationStepConfig( + model_name=topic_cfg.task_metadata.model_name, + provider_name=topic_cfg.task_metadata.model_provider_name, + prompt=topic_cfg.prompt, + ), + input_generation_config=SyntheticDataGenerationStepConfig( + model_name=input_cfg.task_metadata.model_name, + provider_name=input_cfg.task_metadata.model_provider_name, + prompt=input_cfg.prompt, + ), + output_generation_config=SyntheticDataGenerationStepConfig( + model_name=output_cfg.task_metadata.model_name, + provider_name=output_cfg.task_metadata.model_provider_name, + prompt=output_cfg.prompt, + ), + ) + # 5. Create the Spec. Multi-turn leaves sdg_session_config unset — + # the operational state lives on the Eval (full_trace + filter_ids). spec = Spec( parent=task, name=request.name, @@ -398,23 +536,7 @@ async def create_spec_with_copilot( tags=[], eval_id=eval.id, task_sample=request.task_sample, - synthetic_data_generation_session_config=SyntheticDataGenerationSessionConfig( - topic_generation_config=SyntheticDataGenerationStepConfig( - model_name=topic_generation_config.task_metadata.model_name, - provider_name=topic_generation_config.task_metadata.model_provider_name, - prompt=topic_generation_config.prompt, - ), - input_generation_config=SyntheticDataGenerationStepConfig( - model_name=input_generation_config.task_metadata.model_name, - provider_name=input_generation_config.task_metadata.model_provider_name, - prompt=input_generation_config.prompt, - ), - output_generation_config=SyntheticDataGenerationStepConfig( - model_name=output_generation_config.task_metadata.model_name, - provider_name=output_generation_config.task_metadata.model_provider_name, - prompt=output_generation_config.prompt, - ), - ), + synthetic_data_generation_session_config=sdg_session_config_for_spec, ) models_to_save.append(spec) @@ -431,10 +553,19 @@ async def create_spec_with_copilot( for run in task_runs: run.save_to_file() saved_models.append(run) - dataset_runs.save_pending_feedback(run) + if dataset_runs is not None: + dataset_runs.save_pending_feedback(run) spec.save_to_file() saved_models.append(spec) + + # Multi-turn: tag existing chain leaves with eval/golden filter + # tags AFTER spec has saved. Tag-add is idempotent (set semantics); + # leaves persisted to disk pick up the new tags on next read. + # Failure here would leave the spec without a populated dataset — + # surface it so the cleanup path runs. + if request.multi_turn is not None: + tag_multi_turn_chains_for_eval(multi_turn_leaves, eval_tag, golden_tag) except Exception: # Clean up any models that were successfully saved before the error for model in reversed(saved_models): diff --git a/app/desktop/studio_server/test_copilot_api.py b/app/desktop/studio_server/test_copilot_api.py index a7983c6e70..d54ba81894 100644 --- a/app/desktop/studio_server/test_copilot_api.py +++ b/app/desktop/studio_server/test_copilot_api.py @@ -14,8 +14,10 @@ from app.desktop.studio_server.utils.copilot_utils import DatasetTaskRuns from fastapi import FastAPI from fastapi.testclient import TestClient -from kiln_ai.datamodel import Project, Task +from kiln_ai.datamodel import Project, Task, TaskRun +from kiln_ai.datamodel.eval import EvalDataType from kiln_ai.datamodel.spec_properties import SpecType +from kiln_ai.datamodel.task_output import DataSource, DataSourceType, TaskOutput from kiln_server.custom_errors import connect_custom_errors @@ -445,3 +447,236 @@ def test_create_spec_with_copilot_success( specs = task.specs() assert len(specs) == 1 assert specs[0].eval_id == evals[0].id + + +class TestClassifySpecDescription: + """Stub endpoint. Returns 501 until kiln_server ships the real classifier.""" + + def test_returns_501(self, client): + response = client.post( + "/api/copilot/classify_spec_description", + json={"description": "A classification request"}, + ) + + assert response.status_code == 501 + + +class TestCreateSpecWithCopilotMultiTurn: + """Multi-turn save path: tag existing chain leaves instead of synthesising + new examples. See specs/projects/eval_builder_v2/design.md for context. + """ + + BATCH_TAG = "abc123def456" + + @pytest.fixture + def project_and_task(self, tmp_path): + project_path = tmp_path / "test_project" / "project.kiln" + project_path.parent.mkdir() + project = Project(name="Test Project", path=project_path) + project.save_to_file() + task = Task( + name="Test Task", + instruction="Test instruction", + description="Test task", + parent=project, + ) + task.save_to_file() + return project, task + + @pytest.fixture + def synthetic_chain_leaves(self, project_and_task): + """Persist three single-run "chains" tagged like the multi-turn runner + leaves them. Single TaskRuns (no actual multi-turn parents) are + sufficient: the endpoint only cares about the leaf tag. + """ + _, task = project_and_task + source = DataSource( + type=DataSourceType.synthetic, + properties={ + "model_name": "haiku", + "model_provider": "openrouter", + "adapter_name": "kiln_synthetic_user_runner", + }, + ) + leaves = [] + for i in range(3): + run = TaskRun( + parent=task, + input=f"input {i}", + input_source=source, + output=TaskOutput(output=f"output {i}", source=source), + tags=[ + "synthetic_user_case", + f"synthetic_user_batch:{TestCreateSpecWithCopilotMultiTurn.BATCH_TAG}", + ], + ) + run.save_to_file() + leaves.append(run) + return leaves + + @pytest.fixture + def multi_turn_request_data(self): + step_config = { + "task_metadata": { + "model_name": "gpt-4", + "model_provider_name": "openai", + }, + "prompt": "Test prompt", + } + return { + "name": "Multi Turn Spec", + "definition": "The agent should not fabricate policies", + "properties": { + "spec_type": SpecType.issue.value, + "issue_description": "Don't make stuff up", + }, + "evaluate_full_trace": True, + "judge_info": step_config, + "multi_turn": {"batch_tag": TestCreateSpecWithCopilotMultiTurn.BATCH_TAG}, + "task_description": "Test task", + "task_prompt_with_example": "Test prompt", + } + + def test_multi_turn_save_success_tags_chains_and_creates_eval( + self, + client, + project_and_task, + synthetic_chain_leaves, + multi_turn_request_data, + ): + project, task = project_and_task + + with ( + patch( + "app.desktop.studio_server.copilot_api.task_from_id", + return_value=task, + ), + patch( + "app.desktop.studio_server.copilot_api.generate_memorable_name", + return_value="multi-turn-judge", + ), + ): + response = client.post( + f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot", + json=multi_turn_request_data, + ) + + assert response.status_code == 200, response.text + res = response.json() + assert res["name"] == "Multi Turn Spec" + assert res["eval_id"] is not None + # Multi-turn doesn't snapshot a generation config on the spec — + # the operational state lives on the Eval. + assert res["synthetic_data_generation_session_config"] is None + + # Eval: full_trace data type + judge config attached. + # Note: train_set_filter_id is saved as None on disk but a + # backward-compat migration in Eval (libs/core/.../eval.py + # migrate_train_set_filter_id) auto-populates it to + # tag::train_ on read for legacy evals. The functional + # invariant for multi-turn is "no runs tagged with train_*", + # checked below — the field value itself is incidental. + evals = task.evals() + assert len(evals) == 1 + eval_obj = evals[0] + assert eval_obj.evaluation_data_type == EvalDataType.full_trace + assert eval_obj.eval_set_filter_id == "tag::eval_multi_turn_spec" + assert eval_obj.current_config_id is not None + + # Leaves got the eval + golden tags applied on top of their existing + # synthetic_user_* tags. No train tag (multi-turn has no train set). + expected_eval_tag = "eval_multi_turn_spec" + expected_golden_tag = "eval_golden_multi_turn_spec" + train_tag = "train_multi_turn_spec" + for leaf in task.runs(): + assert expected_eval_tag in leaf.tags + assert expected_golden_tag in leaf.tags + assert train_tag not in leaf.tags + assert "synthetic_user_case" in leaf.tags + + def test_multi_turn_save_404_when_batch_tag_matches_nothing( + self, client, project_and_task, multi_turn_request_data + ): + project, task = project_and_task + + with patch( + "app.desktop.studio_server.copilot_api.task_from_id", + return_value=task, + ): + response = client.post( + f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot", + json=multi_turn_request_data, + ) + + assert response.status_code == 404 + assert "batch_tag" in response.json()["message"] + # No models created when the lookup fails up front. + assert len(task.evals()) == 0 + assert len(task.specs()) == 0 + + def test_validator_rejects_both_multi_turn_and_sdg_config( + self, client, project_and_task, multi_turn_request_data + ): + project, task = project_and_task + step_config = { + "task_metadata": { + "model_name": "gpt-4", + "model_provider_name": "openai", + }, + "prompt": "Test prompt", + } + multi_turn_request_data["sdg_session_config"] = { + "topic_generation_config": step_config, + "input_generation_config": step_config, + "output_generation_config": step_config, + } + + with patch( + "app.desktop.studio_server.copilot_api.task_from_id", + return_value=task, + ): + response = client.post( + f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot", + json=multi_turn_request_data, + ) + + assert response.status_code == 422 + body = response.json() + # Pydantic surfaces the validator message somewhere in the response. + assert "multi_turn" in str(body) and "sdg_session_config" in str(body) + + def test_validator_rejects_neither_multi_turn_nor_sdg_config( + self, client, project_and_task, multi_turn_request_data + ): + project, task = project_and_task + del multi_turn_request_data["multi_turn"] + + with patch( + "app.desktop.studio_server.copilot_api.task_from_id", + return_value=task, + ): + response = client.post( + f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot", + json=multi_turn_request_data, + ) + + assert response.status_code == 422 + assert "multi_turn" in str(response.json()) + + def test_validator_rejects_multi_turn_without_evaluate_full_trace( + self, client, project_and_task, multi_turn_request_data + ): + project, task = project_and_task + multi_turn_request_data["evaluate_full_trace"] = False + + with patch( + "app.desktop.studio_server.copilot_api.task_from_id", + return_value=task, + ): + response = client.post( + f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot", + json=multi_turn_request_data, + ) + + assert response.status_code == 422 + assert "evaluate_full_trace" in str(response.json()) diff --git a/app/desktop/studio_server/utils/copilot_utils.py b/app/desktop/studio_server/utils/copilot_utils.py index 5aef604565..fc7f71f31e 100644 --- a/app/desktop/studio_server/utils/copilot_utils.py +++ b/app/desktop/studio_server/utils/copilot_utils.py @@ -25,7 +25,7 @@ ) from app.desktop.studio_server.utils.response_utils import unwrap_response from fastapi import HTTPException -from kiln_ai.datamodel import Feedback, FeedbackSource, TaskRun +from kiln_ai.datamodel import Feedback, FeedbackSource, Task, TaskRun from kiln_ai.datamodel.datamodel_enums import TaskOutputRatingType from kiln_ai.datamodel.task_output import ( DataSource, @@ -36,6 +36,12 @@ ) from kiln_ai.utils.config import Config +# Tag prefix the multi-turn synthetic-user runner stamps on each chain's leaf +# TaskRun — see kiln_ai.synthetic_user.runner._TAG_PREFIX_SU_BATCH. Kept in +# sync manually; if the runner ever changes its tag scheme this constant +# moves too. +_TAG_PREFIX_SU_BATCH = "synthetic_user_batch:" + # Constants for copilot spec creation KILN_COPILOT_MODEL_NAME = "kiln-copilot" KILN_COPILOT_MODEL_PROVIDER = "kiln" @@ -298,3 +304,39 @@ def create_dataset_task_runs( result.add_run(create_task_run_from_sample(example, train_tag, extra_tags)) return result + + +def find_multi_turn_chain_leaves(task: Task, batch_tag: str) -> list[TaskRun]: + """Return the leaf TaskRuns of all chains tagged with the given batch_tag. + + The multi-turn runner tags only the leaf of each chain with + "synthetic_user_batch:{batch_tag}". Walking parent_task_run_id from the + leaf reconstructs the full conversation if a caller needs it; for eval + purposes the leaf alone is enough because its `.trace` field already + holds the cumulative OpenAI-format conversation. + """ + target_tag = f"{_TAG_PREFIX_SU_BATCH}{batch_tag}" + return [run for run in task.runs() if target_tag in (run.tags or [])] + + +def tag_multi_turn_chains_for_eval( + leaves: list[TaskRun], + eval_tag: str, + golden_tag: str, +) -> None: + """Add eval and golden filter tags to existing chain leaves. + + Tags are deduplicated. We tag every leaf with BOTH eval and golden tags + so the chain is part of the eval dataset AND the golden ratings set — + multi-turn batches are small (≤10 chains) so there's no meaningful + split. No train tag is applied (multi-turn evals don't have a train + set in MVP — see specs/projects/eval_builder_v2/design.md). + + Mutates each leaf in place and persists via save_to_file. + """ + for leaf in leaves: + tags = set(leaf.tags or []) + tags.add(eval_tag) + tags.add(golden_tag) + leaf.tags = sorted(tags) + leaf.save_to_file() diff --git a/app/web_ui/src/lib/api_schema.d.ts b/app/web_ui/src/lib/api_schema.d.ts index f05dd26d78..60effa1a29 100644 --- a/app/web_ui/src/lib/api_schema.d.ts +++ b/app/web_ui/src/lib/api_schema.d.ts @@ -2644,6 +2644,30 @@ export interface paths { patch?: never; trace?: never; }; + "/api/copilot/classify_spec_description": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Classify Spec Description + * @description Classify a free-text spec description into one of the 9 spec types. + * + * Not implemented yet — the underlying kiln_server classifier hasn't + * shipped. Returns 501 so callers can detect this and fall back to + * manual spec-type selection. + */ + post: operations["classify_spec_description_api_copilot_classify_spec_description_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/copilot/clarify_spec": { parameters: { query?: never; @@ -4106,6 +4130,48 @@ export interface components { judge_result: components["schemas"]["SyntheticDataGenerationStepConfigApi-Output"]; sdg_session_config: components["schemas"]["SyntheticDataGenerationSessionConfigApi-Output"]; }; + /** + * ClassifySpecDescriptionInput + * @description Free-text description of an eval the user wants to build. The + * endpoint maps it to one of the 9 spec types and pre-fills the + * property_values for that type so the v2 builder can skip the + * template-carousel step entirely. + */ + ClassifySpecDescriptionInput: { + /** + * Description + * @description Free-text description of what the eval should check. + */ + description: string; + /** + * Task Prompt + * @description Optional task prompt for context (improves classification accuracy when the spec relates to a specific task). + */ + task_prompt?: string | null; + }; + /** + * ClassifySpecDescriptionOutput + * @description Classified spec type + suggested name + spec_type-specific property + * values. The `property_values` dict keys match the FieldConfig keys + * defined for the chosen spec_type (see + * app/web_ui/.../select_template/spec_templates.ts). + */ + ClassifySpecDescriptionOutput: { + /** @description The classified spec type. */ + spec_type: components["schemas"]["SpecType"]; + /** + * Suggested Name + * @description A filename-safe name for the new spec, derived from the description. + */ + suggested_name: string; + /** + * Property Values + * @description Pre-filled property values for the chosen spec_type. Keys correspond to the field_configs of that spec_type. + */ + property_values: { + [key: string]: string; + }; + }; /** * CloneRequest * @description Request to clone a git repository into a temporary directory. @@ -4522,17 +4588,22 @@ export interface components { * CreateSpecWithCopilotRequest * @description Request model for creating a spec with Kiln Copilot. * - * This endpoint uses Kiln Copilot to: - * - Generate batch examples for eval, train, and golden datasets - * - Create a judge eval config - * - Create an eval with appropriate template/output scores - * - Create and save the spec + * Two synthesis paths are supported, exactly one must be set per request: + * + * - **Single-turn:** caller supplies `sdg_session_config`. Endpoint calls + * `generate_copilot_examples` for fresh I/O pairs, splits them into + * eval/train/golden datasets, and tags new TaskRuns. * - * If you don't want to use copilot, use the regular POST /spec endpoint instead. + * - **Multi-turn:** caller supplies `multi_turn` with a `batch_tag` pointing + * at chains already on disk (created earlier by the synthetic-user runner). + * Endpoint tags the existing chain leaves with eval/golden filter tags; + * no new TaskRuns are created. `evaluate_full_trace` must be True. + * + * If you don't want copilot at all, use POST /spec instead. * * The client is responsible for building: - * - definition: The spec definition string (use buildSpecDefinition on client) - * - properties: The spec properties object (filtered, with spec_type included) + * - definition: the spec definition string (buildSpecDefinition on client) + * - properties: the spec properties object (filtered, with spec_type included) */ CreateSpecWithCopilotRequest: { /** Name */ @@ -4555,7 +4626,8 @@ export interface components { /** Reviewed Examples */ reviewed_examples?: components["schemas"]["ReviewedExample"][]; judge_info: components["schemas"]["SyntheticDataGenerationStepConfigApi-Input"]; - sdg_session_config: components["schemas"]["SyntheticDataGenerationSessionConfigApi-Input"]; + sdg_session_config?: components["schemas"]["SyntheticDataGenerationSessionConfigApi-Input"] | null; + multi_turn?: components["schemas"]["MultiTurnSaveInfo"] | null; /** * Task Description * @default @@ -7430,6 +7502,19 @@ export interface components { * @enum {string} */ ModelProviderName: "openai" | "groq" | "amazon_bedrock" | "ollama" | "openrouter" | "fireworks_ai" | "kiln_fine_tune" | "kiln_custom_registry" | "openai_compatible" | "anthropic" | "gemini_api" | "azure_openai" | "huggingface" | "vertex" | "together_ai" | "siliconflow_cn" | "cerebras" | "docker_model_runner"; + /** + * MultiTurnSaveInfo + * @description Identifies an existing multi-turn synthetic-user batch to turn into an Eval. + * The endpoint walks chains tagged with this batch_tag and applies eval/golden + * filter tags instead of generating new examples. + */ + MultiTurnSaveInfo: { + /** + * Batch Tag + * @description The batch_tag emitted by the multi-turn synthetic-user runner (see kiln_ai.synthetic_user.runner). Identifies the set of conversation chains already persisted to disk that this Eval should evaluate. + */ + batch_tag: string; + }; /** * NewProposedSpecEditApi * @description A proposed edit to a spec field. @@ -9216,6 +9301,12 @@ export interface components { * @enum {string} */ SpecStatus: "active" | "future" | "deprecated" | "archived"; + /** + * SpecType + * @description Defines the type of spec. + * @enum {string} + */ + SpecType: "desired_behaviour" | "issue" | "tone" | "formatting" | "localization" | "appropriate_tool_use" | "reference_answer_accuracy" | "factual_correctness" | "hallucinations" | "completeness" | "toxicity" | "bias" | "maliciousness" | "nsfw" | "taboo" | "jailbreak" | "prompt_leakage"; /** * SpecificationInput * @description The specification to refine. @@ -16835,6 +16926,39 @@ export interface operations { }; }; }; + classify_spec_description_api_copilot_classify_spec_description_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["ClassifySpecDescriptionInput"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ClassifySpecDescriptionOutput"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; clarify_spec_api_copilot_clarify_spec_post: { parameters: { query?: never; From 4cb609f83488bb4b9c110a3519eebece6c2858f7 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:04:01 -0700 Subject: [PATCH 18/36] feat(specs-v2): mock-first eval builder UI (single-turn + multi-turn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the v2 eval builder at /specs_v2/{project_id}/{task_id} — a single- page wizard that simplifies v1's 9-template carousel + multi-field form into one description box → Q&A → editable refine → generate → review → save flow. Sidebar entry under "Evals V2 (Beta)". Reuses v1 spec_builder components (Questions, RefineSpec, ReviewExamples) on the shared screens so the look-and-feel stays in sync with v1. The multi-turn branch at Step 4 + 5 + 6 has its own custom UI for chat-trace review and ties into the new spec_with_copilot multi-turn save path. Step 1 calls classify_spec_description (currently 501) with a graceful fallback to "issue" defaults. Multi-turn generation (Step 4) is a stub — real run_cases_batch SSE wiring is still pending; surface a clear error until that lands. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/web_ui/src/lib/ui/section.ts | 1 + app/web_ui/src/routes/(app)/+layout.svelte | 16 + .../[project_id]/[task_id]/+page.svelte | 863 ++++++++++++++++++ .../specs_v2/[project_id]/[task_id]/+page.ts | 1 + 4 files changed, 881 insertions(+) create mode 100644 app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte create mode 100644 app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts diff --git a/app/web_ui/src/lib/ui/section.ts b/app/web_ui/src/lib/ui/section.ts index 0dd7728472..437ec26a2e 100644 --- a/app/web_ui/src/lib/ui/section.ts +++ b/app/web_ui/src/lib/ui/section.ts @@ -4,6 +4,7 @@ export enum Section { Settings, Prompts, Specs, + SpecsV2, Generate, Run, FineTune, diff --git a/app/web_ui/src/routes/(app)/+layout.svelte b/app/web_ui/src/routes/(app)/+layout.svelte index ac367455bb..80971d46e8 100644 --- a/app/web_ui/src/routes/(app)/+layout.svelte +++ b/app/web_ui/src/routes/(app)/+layout.svelte @@ -104,6 +104,8 @@ section = Section.Documents } else if (path_start("/models", $page.url.pathname)) { section = Section.Models + } else if (path_start("/specs_v2", $page.url.pathname)) { + section = Section.SpecsV2 } else if (path_start("/specs", $page.url.pathname)) { section = Section.Specs } else if (path_start("/optimize", $page.url.pathname)) { @@ -276,6 +278,20 @@ > +
  • + +
    + +
    + + Evals V2 +
    Beta
    +     +
    • +
  • + import AppPage from "../../../app_page.svelte" + import { page } from "$app/stores" + import { onMount } from "svelte" + import { agentInfo } from "$lib/agent" + import { goto } from "$app/navigation" + import { client } from "$lib/api_client" + import FormElement from "$lib/utils/form_element.svelte" + import { load_task } from "$lib/stores" + // Reuse v1 spec_builder components so v2 looks identical on the shared + // screens (clarify Q&A, refine). When v1 evolves, v2 follows for free. + import Questions from "../../../specs/[project_id]/[task_id]/spec_builder/questions.svelte" + import RefineSpec from "../../../specs/[project_id]/[task_id]/spec_builder/refine_spec.svelte" + import ReviewExamples from "../../../specs/[project_id]/[task_id]/spec_builder/review_examples.svelte" + import { spec_field_configs } from "../../../specs/[project_id]/[task_id]/select_template/spec_templates" + import type { + SuggestedEdit, + ReviewRow, + } from "../../../specs/[project_id]/[task_id]/spec_utils" + import { KilnError } from "$lib/utils/error_handlers" + import type { + Task, + QuestionSet, + QuestionWithAnswer, + SpecType, + SubsampleBatchOutputItemApi, + SyntheticDataGenerationSessionConfigApi, + SyntheticDataGenerationStepConfigApi, + ReviewedExample, + } from "$lib/types" + + $: project_id = $page.params.project_id! + $: task_id = $page.params.task_id! + $: agentInfo.set({ + name: "Evals V2", + description: `V2 eval builder for project ${project_id}, task ${task_id} — single-page wizard for spec authoring with multi-turn support.`, + }) + + // ── State machine for the v2 builder. + // describe — Step 1: free-text "what to evaluate" + // clarify — Step 2: Q&A (no live preview — fast collection) + // refine — Step 3: editable proposed spec edits (mirrors v1's refine screen) + // generate — Step 4: single-turn examples or multi-turn chains + // review — Step 5: pass/fail review + suggested spec refinements + // save — Step 6: persist Spec + Eval + EvalConfig + dataset + type BuilderStep = + | "describe" + | "clarify" + | "refine" + | "generate" + | "review" + | "save" + | "done" + let current_step: BuilderStep = "describe" + const TOTAL_STEPS = 6 + const STEP_INDEX: Record = { + describe: 1, + clarify: 2, + refine: 3, + generate: 4, + review: 5, + save: 6, + done: 6, + } + + // ── Task (needed to know turn_mode for the Step 3 branch) + let task: Task | null = null + let task_loading = true + let task_error: string | null = null + $: is_multi_turn = task?.turn_mode === "multiturn" + + onMount(async () => { + try { + task = await load_task(project_id, task_id) + } catch (e) { + task_error = e instanceof Error ? e.message : "Failed to load task." + } finally { + task_loading = false + } + }) + + // ── Step 1 state + let description = "" + // classify_spec_description should overwrite spec_type / name / + // property_values when the kiln_server classifier ships. Defaulting to + // "issue" keeps refine + save shapes valid in the meantime. + let spec_type: SpecType = "issue" + let name = "" + let property_values: Record = { + issue_description: "", + issue_examples: "", + non_issue_examples: "", + } + let classifying = false + let classify_error: string | null = null + $: field_configs = spec_field_configs[spec_type] + + // Call classify_spec_description to map the free-text Step 1 description + // to a spec_type + suggested name + structured property_values. The + // endpoint currently returns 501 — on error we keep the "issue" defaults + // so the user can still proceed and fill in property_values via the + // Q&A / Refine steps. + async function classify_then_continue() { + classifying = true + classify_error = null + try { + const { data, error } = await client.POST( + "/api/copilot/classify_spec_description", + { + body: { + description, + task_prompt: task?.instruction ?? null, + }, + }, + ) + if (error || !data) { + classify_error = + "Couldn't classify your description — continuing with default 'issue' type." + current_step = "clarify" + return + } + spec_type = data.spec_type as SpecType + name = data.suggested_name + // The classifier returns the property_values dict already keyed for + // this spec_type. Cast to the looser Record shape consumed by + // Questions / RefineSpec. + property_values = data.property_values as Record + current_step = "clarify" + } catch (e) { + classify_error = + e instanceof Error ? e.message : "Couldn't classify your description." + } finally { + classifying = false + } + } + + // ── Step 2 state — questions + let question_set: QuestionSet | null = null + let questions_loading = false + let questions_error: string | null = null + let questions_form_error: KilnError | null = null + let questions_submitting = false + // Bound to the Questions component so selections survive remounts when + // user navigates back from Refine to Clarify. + let selections: (number | "other" | null)[] = [] + let other_texts: string[] = [] + + async function load_questions() { + questions_loading = true + questions_error = null + try { + const { data, error } = await client.POST("/api/copilot/question_spec", { + body: { + target_task_info: { + task_prompt: task?.instruction ?? "", + task_input_schema: "", + task_output_schema: "", + }, + target_specification: description, + }, + }) + if (error || !data) { + questions_error = "Failed to load clarifying questions." + return + } + question_set = data as QuestionSet + selections = question_set.questions.map(() => null) + other_texts = question_set.questions.map(() => "") + } catch (e) { + questions_error = + e instanceof Error ? e.message : "Failed to load questions." + } finally { + questions_loading = false + } + } + + // ── Step 3 state — refine (shape required by v1's RefineSpec component) + let refined_property_values: Record = {} + let suggested_edits: Record = {} + let not_incorporated_feedback: string = "" + let refine_form_error: KilnError | null = null + let refine_submitting = false + let refined_preview_loading = false + + // Called by Questions component on Continue. Fires the refinement call and + // populates the state shape consumed by RefineSpec. Matches v1's flow: + // answer Qs → refining spinner → refine screen with editable suggestions. + async function on_continue_from_clarify( + questions_and_answers: QuestionWithAnswer[], + ) { + current_step = "refine" + refined_preview_loading = true + try { + const spec_fields: Record = {} + const spec_field_current_values: Record = {} + for (const field of field_configs) { + spec_fields[field.key] = field.description + spec_field_current_values[field.key] = property_values[field.key] ?? "" + } + + const { data, error } = await client.POST( + "/api/copilot/refine_spec_with_question_answers", + { + body: { + task_prompt: task?.instruction ?? "", + specification: { spec_fields, spec_field_current_values }, + questions_and_answers, + }, + }, + ) + if (error || !data) return + + const refine_response = data as { + new_proposed_spec_edits?: { + spec_field_name: string + proposed_edit: string + reason_for_edit?: string + }[] + not_incorporated_feedback?: string + } + + // Start from current values, then apply each proposed edit. Mirrors v1's + // processProposedSpecEdits helper in spec_builder/+page.svelte. + const refined = { ...property_values } + const edits: Record = {} + for (const edit of refine_response.new_proposed_spec_edits ?? []) { + refined[edit.spec_field_name] = edit.proposed_edit + edits[edit.spec_field_name] = { + proposed_value: edit.proposed_edit, + reason_for_edit: edit.reason_for_edit ?? "", + } + } + refined_property_values = refined + suggested_edits = edits + not_incorporated_feedback = + refine_response.not_incorporated_feedback ?? "" + } catch { + // Silent — the refine response is optional; on error we land on the + // refine step with no suggested edits and the user can advance. + } finally { + refined_preview_loading = false + } + } + + // Both events from RefineSpec — analyze_refined (user edited something) + // and create_spec (no edits) — advance to Step 4 generation. v2 doesn't + // re-analyze, it just uses whatever refined_property_values the user + // finalized. + function on_refine_submit() { + on_advance_to_generate() + } + + // ── Step 3 state — generation + let generation_loading = false + let generation_error: string | null = null + // single-turn output + let single_turn_examples: SubsampleBatchOutputItemApi[] = [] + let sdg_session_config: SyntheticDataGenerationSessionConfigApi | null = null + let judge_info: SyntheticDataGenerationStepConfigApi | null = null + + // multi-turn output — chains populated from real run_cases_batch SSE + // events once that wiring lands. Until then, on_generate_multi_turn + // surfaces an "endpoint wiring pending" error. + type ChainTurn = { role: "user" | "assistant"; content: string } + type Chain = { + case_index: number + persona_summary: string + total_cost: number + trace: ChainTurn[] + } + // Number of synthetic-user cases to drive in one multi-turn batch — + // matches NUM_CASES_MAX in libs/core/kiln_ai/synthetic_user/runner.py. + const NUM_CASES = 10 + let multi_turn_chains: Chain[] = [] + let multi_turn_progress = 0 // 0..N as cases complete + const multi_turn_total = NUM_CASES + // Real batch_tag from run_cases_batch's BatchStartedEvent. The on_save + // multi-turn branch uses this to tell the backend which chains to tag + // for the eval dataset. Null until SSE wiring is added. + let real_multi_turn_batch_tag: string | null = null + + async function on_generate_single_turn() { + generation_loading = true + generation_error = null + try { + const { data, error } = await client.POST("/api/copilot/clarify_spec", { + body: { + target_task_info: { + task_prompt: task?.instruction ?? "", + task_input_schema: "", + task_output_schema: "", + }, + target_specification: description, + num_samples_per_topic: 10, + num_topics: 10, + providers: ["openrouter"], + num_exemplars: 10, + }, + }) + if (error || !data) { + generation_error = "Failed to generate examples." + return + } + single_turn_examples = data.examples_for_feedback ?? [] + sdg_session_config = data.sdg_session_config ?? null + judge_info = data.judge_result ?? null + current_step = "review" + } catch (e) { + generation_error = e instanceof Error ? e.message : "Generation failed." + } finally { + generation_loading = false + } + } + + async function on_generate_multi_turn() { + // TODO: wire EventSource against + // POST /api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch + // Dispatch on BatchStartedEvent → seed real_multi_turn_batch_tag, + // TurnCompletedEvent → update progress, CaseCompletedEvent → append + // to multi_turn_chains, BatchCompletedEvent → advance step. + // Until that wiring lands the multi-turn pipeline can't produce real + // chains — surface a clear error rather than silently doing nothing. + generation_loading = true + generation_error = null + multi_turn_progress = 0 + multi_turn_chains = [] + await sleep(100) + generation_error = + "Multi-turn generation isn't wired to run_cases_batch yet." + generation_loading = false + } + + function on_continue_from_generate_step() { + if (is_multi_turn) { + on_generate_multi_turn() + } else { + on_generate_single_turn() + } + } + + // Advance from the Refine step (3) into Generate (4) and immediately kick + // off generation — no extra click required. The in-step button only + // surfaces if generation errored, as a retry affordance. + function on_advance_to_generate() { + current_step = "generate" + on_continue_from_generate_step() + } + + // Same pattern for Review (5) → Save (6): land on Save with the request + // already in flight; only show the in-step button on error as retry. + function on_advance_to_save() { + current_step = "save" + on_save() + } + + // ── Step 5 state — single-turn review (uses v1's ReviewExamples) + // ReviewRow is the shape ReviewExamples reads/mutates internally — + // it bumps user_says_meets_spec + feedback on each pass/fail click. + let review_rows: ReviewRow[] = [] + let review_form_error: KilnError | null = null + let review_submitting = false + // Convert generation output to ReviewRow shape when examples become + // available (Step 4 → Step 5 transition). Only initializes — subsequent + // user edits stay on the array. + $: if ( + single_turn_examples.length > 0 && + review_rows.length !== single_turn_examples.length + ) { + review_rows = single_turn_examples.map((e, i) => ({ + input: e.input, + output: e.output, + model_says_meets_spec: !e.fails_specification, + feedback: "", + row_id: `row_${i}`, + })) + } + + // Multi-turn: per-chain pass/fail + let chain_verdicts: Array<{ + verdict: "pass" | "fail" | null + feedback: string + }> = [] + $: chain_verdicts = + multi_turn_chains.length > 0 && + chain_verdicts.length !== multi_turn_chains.length + ? multi_turn_chains.map(() => ({ verdict: null, feedback: "" })) + : chain_verdicts + + // ── Step 6 state — save + let saving = false + let save_error: string | null = null + + // Fetch a default judge_info via clarify_spec for the multi-turn save + // path. The clarify_spec response also includes single-turn examples and + // sdg_session_config which we ignore — multi-turn save uses neither. + // Stopgap until a dedicated "default judge config" endpoint ships. + async function fetch_judge_info_via_clarify_spec() { + const { data, error } = await client.POST("/api/copilot/clarify_spec", { + body: { + target_task_info: { + task_prompt: task?.instruction ?? "", + task_input_schema: "", + task_output_schema: "", + }, + target_specification: description, + num_samples_per_topic: 10, + num_topics: 10, + providers: ["openrouter"], + num_exemplars: 10, + }, + }) + if (error || !data) return false + judge_info = data.judge_result ?? null + return judge_info !== null + } + + async function on_save() { + saving = true + save_error = null + try { + // Source of truth for the saved spec is refined_property_values — + // populated from Step 1 description initially, then updated in Step 3 + // via v1's RefineSpec component when the user accepts/edits the LLM's + // proposed refinements. Fall back to property_values if Step 3 was + // skipped (no refinements were proposed). + const final_values = + Object.keys(refined_property_values).length > 0 + ? refined_property_values + : property_values + const issue_description = final_values.issue_description ?? description + const filtered = Object.fromEntries( + Object.entries(final_values).filter( + ([_, v]) => v !== null && v !== "" && (v ?? "").trim() !== "", + ), + ) + const spec_properties = { + spec_type: "issue" as const, + ...filtered, + issue_description, + } + + if (is_multi_turn) { + if (real_multi_turn_batch_tag === null) { + save_error = + "Multi-turn save needs a batch_tag from run_cases_batch; " + + "generation isn't wired yet." + return + } + const ok = await fetch_judge_info_via_clarify_spec() + if (!ok || !judge_info) { + save_error = "Save failed — couldn't fetch judge config." + return + } + const { data, error } = await client.POST( + "/api/projects/{project_id}/tasks/{task_id}/spec_with_copilot", + { + params: { path: { project_id, task_id } }, + body: { + name, + definition: issue_description, + properties: spec_properties, + evaluate_full_trace: true, + reviewed_examples: [], + judge_info, + multi_turn: { batch_tag: real_multi_turn_batch_tag }, + task_description: "", + task_prompt_with_example: task?.instruction ?? "", + }, + }, + ) + if (error || !data) { + save_error = "Save failed — try again." + return + } + const saved = data as { id?: string } + if (saved.id) { + goto(`/specs/${project_id}/${task_id}/${saved.id}`) + } else { + current_step = "done" + } + return + } + + // Single-turn save path. + const reviewed_examples: ReviewedExample[] = review_rows + .filter((row) => row.user_says_meets_spec !== undefined) + .map((row) => ({ + input: row.input, + output: row.output, + model_says_meets_spec: row.model_says_meets_spec, + user_says_meets_spec: row.user_says_meets_spec as boolean, + feedback: row.feedback, + })) + + if (!sdg_session_config || !judge_info) { + save_error = + "Missing generation config — go back to Step 4 and regenerate." + return + } + + const { data, error } = await client.POST( + "/api/projects/{project_id}/tasks/{task_id}/spec_with_copilot", + { + params: { path: { project_id, task_id } }, + body: { + name, + definition: issue_description, + properties: spec_properties, + evaluate_full_trace: false, + reviewed_examples, + judge_info, + sdg_session_config, + task_description: "", + task_prompt_with_example: task?.instruction ?? "", + }, + }, + ) + if (error || !data) { + save_error = "Save failed — try again." + return + } + // Land on the spec/eval detail page (titled "Eval: ..."). This is + // the same destination v1 uses. + const saved = data as { id?: string } + if (saved.id) { + goto(`/specs/${project_id}/${task_id}/${saved.id}`) + } else { + current_step = "done" + } + } catch (e) { + save_error = e instanceof Error ? e.message : "Save failed." + } finally { + saving = false + } + } + + // ── Navigation helpers + function back_to_task() { + goto(`/specs/${project_id}/${task_id}`) + } + + function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) + } + + // Auto-load questions when entering Step 2 + $: if (current_step === "clarify" && !question_set && !questions_loading) { + load_questions() + } + + + +
    + +
    + Step + {STEP_INDEX[current_step]} + of {TOTAL_STEPS} + {#if is_multi_turn} + multi-turn + {/if} +
    + + {#if task_loading} +
    Loading task…
    + {:else if task_error} +
    {task_error}
    + {:else if current_step === "describe"} + +

    What should this eval check?

    +

    + Describe in plain language. We'll structure it for you. +

    + + + +
    + Spec name (auto-derived after Continue) +
    + + {#if classify_error} +
    {classify_error}
    + {/if} + +
    + + +
    + {:else if current_step === "clarify"} + + {#if questions_loading} +
    + +
    Generating clarifying questions…
    +
    + {:else if questions_error} +
    {questions_error}
    + {:else if question_set} + +
    + +
    + {/if} + {:else if current_step === "refine"} + + {#if refined_preview_loading} +
    + +
    Refining your eval…
    +
    + {:else} + +
    + +
    + {/if} + {:else if current_step === "generate"} + +

    + {#if is_multi_turn} + Running synthetic conversations + {:else} + Generating examples + {/if} +

    +

    + {#if is_multi_turn} + Driving {multi_turn_total} cases against your agent. This will take a moment. + {:else} + Generating ~10 sample inputs and outputs based on your spec. + {/if} +

    + + {#if generation_loading} +
    + + {#if is_multi_turn} +
    + {multi_turn_progress} of {multi_turn_total} ready +
    + {:else} +
    Generating examples…
    + {/if} +
    + {/if} + + {#if generation_error} +
    {generation_error}
    +
    + +
    + {/if} + +
    + +
    + {:else if current_step === "review"} + + {#if is_multi_turn} +

    Review

    +

    + Read each conversation. Mark each chain Pass or Fail. +

    + +
    + {#each multi_turn_chains as chain, ci} +
    +
    +
    +
    +
    + Conversation {ci + 1} of {multi_turn_chains.length} +
    +
    + {chain.persona_summary} +
    +
    +
    + + +
    +
    + +
    + {#each chain.trace as turn} +
    +
    + {turn.role} +
    +
    {turn.content}
    +
    + {/each} +
    + + {#if chain_verdicts[ci]?.verdict !== null} + + {/if} +
    +
    + {/each} +
    +
    + + +
    + {:else} + + +
    + +
    + {/if} + {:else if current_step === "save"} + +

    Save

    +

    + Creating the Spec, Eval, and dataset… +

    + + {#if saving} +
    + +
    Saving…
    +
    + {:else if save_error} +
    {save_error}
    +
    + +
    + {/if} + +
    + +
    + {:else if current_step === "done"} + +
    +
    ✓
    +

    Spec saved

    + +
    + {/if} +
    +     diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts new file mode 100644 index 0000000000..9786e09d9a --- /dev/null +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts @@ -0,0 +1 @@ +export const prerender = false From 70a4a3fb1dbf9808aa11aa184117d05ec6610b61 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:40:44 -0700 Subject: [PATCH 19/36] feat(specs-v2): wire real run_cases_batch SSE for multi-turn generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 4 multi-turn now drives the real run_cases_batch endpoint instead of simulating with hardcoded chains: 1. Resolve the task's default run config → target_run_config (the drive loop invokes the agent the same way a normal task run would). 2. POST /multiturn_sdg/generate_cases → N=10 synthetic-user cases. 3. POST /multiturn_sdg/run_cases_batch as SSE; consume the stream via fetch + ReadableStream (same pattern as streaming_chat.ts since the endpoint is POST and EventSource is GET-only). 4. Dispatch BatchEvents into component state: batch_started seeds the batch_tag used by the multi-turn save, turn_completed updates the cumulative trace per case, case_completed appends a Chain to the review cards, batch_completed advances to Step 5. SU driver model is hardcoded for MVP (claude_4_5_haiku via openrouter) per design.md; surfacing the choice in the UI is deferred. With this in place the multi-turn save path uses a real batch_tag and the previous "no real chains, route through single-turn pipeline" stopgap can be removed. The save step is now a single-path call to spec_with_copilot with multi_turn={batch_tag}. Sidebar entry now carries a comment noting that the v1 Evals tab is intended to be removed once v2 ships GA. Also adds get_task_composite_id to the edit_task test's $lib/stores mock so the new transitive import doesn't trip an incomplete-mock warning during test runs. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/web_ui/src/routes/(app)/+layout.svelte | 4 + .../[project_id]/[task_id]/+page.svelte | 252 ++++++++++++++++-- .../(setup)/create_task/edit_task.test.ts | 2 + 3 files changed, 235 insertions(+), 23 deletions(-) diff --git a/app/web_ui/src/routes/(app)/+layout.svelte b/app/web_ui/src/routes/(app)/+layout.svelte index 80971d46e8..819aaffd55 100644 --- a/app/web_ui/src/routes/(app)/+layout.svelte +++ b/app/web_ui/src/routes/(app)/+layout.svelte @@ -278,6 +278,10 @@ >
    • +
  • c.id === task!.default_run_config_id, + ) + if (!default_config) { + generation_error = "Task default run config not found." + return + } + const rcp = default_config.run_config_properties + if (!isKilnAgentRunConfig(rcp)) { + generation_error = + "Multi-turn requires a Kiln Agent run config on the task default." + return + } + const target_run_config = { + model_name: rcp.model_name, + model_provider: rcp.model_provider_name, + prompt_id: rcp.prompt_id ?? "simple_prompt_builder", + } + + // 2. Generate synthetic-user cases via copilot. + const refined_description = + (refined_property_values.issue_description as string | null) ?? + description + const cases_resp = await client.POST( + "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/generate_cases", + { + params: { path: { project_id, task_id } }, + body: { + target_specification: refined_description, + num_cases: NUM_CASES, + }, + }, + ) + if (cases_resp.error || !cases_resp.data) { + generation_error = "Failed to generate synthetic-user cases." + return + } + + // 3. Stream run_cases_batch. The endpoint is POST so we can't use + // EventSource (which is GET-only). Manual fetch + ReadableStream + + // SSE line parsing — same pattern as streaming_chat.ts. + const url = `${base_url}/api/projects/${project_id}/tasks/${task_id}/multiturn_sdg/run_cases_batch` + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "text/event-stream", + }, + body: JSON.stringify({ + cases: cases_resp.data.cases, + turns: TURNS_PER_CASE, + target_run_config, + su_driver: SU_DRIVER_DEFAULT, + }), + }) + + if (!response.ok || !response.body) { + let detail: string + try { + const err_json = await response.json() + detail = err_json?.message ?? err_json?.detail?.message ?? "unknown" + } catch { + detail = await response.text().catch(() => "unknown") + } + generation_error = `run_cases_batch failed (${response.status}): ${detail}` + return + } + + // Per-case cumulative trace from turn_completed events. The leaf's + // trace at case_completed time is what we render on the review cards. + const traces_by_case: Record = {} + const reader = response.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + + stream_loop: while (true) { + const { done, value } = await reader.read() + if (done) break stream_loop + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + if (!line.startsWith("data: ")) continue + const payload = line.slice(6).trim() + if (!payload) continue + let event: SseEvent + try { + event = JSON.parse(payload) as SseEvent + } catch { + continue + } + + if (event.event === "batch_started") { + real_multi_turn_batch_tag = event.batch_tag + } else if (event.event === "turn_completed") { + // event.trace is OpenAI-format messages; project to ChainTurn. + const turns: ChainTurn[] = (event.trace ?? []) + .map((msg) => ({ + role: msg.role as ChainTurn["role"], + content: + typeof msg.content === "string" + ? msg.content + : JSON.stringify(msg.content), + })) + .filter( + (t: ChainTurn) => t.role === "user" || t.role === "assistant", + ) + traces_by_case[event.case_index] = turns + } else if (event.event === "case_completed") { + multi_turn_chains = [ + ...multi_turn_chains, + { + case_index: event.case_index, + persona_summary: `Case ${event.case_index + 1}`, + total_cost: event.total_cost ?? 0, + trace: traces_by_case[event.case_index] ?? [], + }, + ] + multi_turn_progress = multi_turn_chains.length + } else if (event.event === "case_failed") { + console.warn( + `SU case ${event.case_index} failed: ${event.error_code} ${event.message}`, + ) + } else if (event.event === "batch_completed") { + break stream_loop + } + } + } + + if (multi_turn_chains.length === 0) { + generation_error = + "All synthetic-user cases failed — check task and SU driver model availability." + return + } + + current_step = "review" + } catch (e) { + generation_error = + e instanceof Error ? e.message : "Multi-turn generation failed." + } finally { + generation_loading = false + } } function on_continue_from_generate_step() { @@ -394,6 +599,8 @@ // path. The clarify_spec response also includes single-turn examples and // sdg_session_config which we ignore — multi-turn save uses neither. // Stopgap until a dedicated "default judge config" endpoint ships. + // clarify_spec is single-turn-specific but its response includes a + // judge_result that's reusable as the multi-turn save's judge_info. async function fetch_judge_info_via_clarify_spec() { const { data, error } = await client.POST("/api/copilot/clarify_spec", { body: { @@ -439,11 +646,14 @@ issue_description, } + // Multi-turn save: tag the chains produced by run_cases_batch. + // judge_info isn't carried through the multi-turn pipeline so we + // fetch one via clarify_spec — wasteful but works until a dedicated + // "default judge config" endpoint ships. if (is_multi_turn) { if (real_multi_turn_batch_tag === null) { save_error = - "Multi-turn save needs a batch_tag from run_cases_batch; " + - "generation isn't wired yet." + "No multi-turn chains were generated — go back to Step 4." return } const ok = await fetch_judge_info_via_clarify_spec() @@ -539,10 +749,6 @@ goto(`/specs/${project_id}/${task_id}`) } - function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) - } - // Auto-load questions when entering Step 2 $: if (current_step === "clarify" && !question_set && !questions_loading) { load_questions() diff --git a/app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/edit_task.test.ts b/app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/edit_task.test.ts index b2f627daa1..95c7a9396d 100644 --- a/app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/edit_task.test.ts +++ b/app/web_ui/src/routes/(fullscreen)/setup/(setup)/create_task/edit_task.test.ts @@ -45,6 +45,8 @@ vi.mock("$lib/stores", () => ({ ui_state, load_current_task: vi.fn(), load_rating_options: vi.fn(), + get_task_composite_id: (project_id: string, task_id: string) => + `${project_id}__${task_id}`, })) const EditTask = (await import("./edit_task.svelte")).default From e4f09d3c3c5049e5856f0b51f7c836ec38bb8c19 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Fri, 5 Jun 2026 01:43:34 -0700 Subject: [PATCH 20/36] fix(specs-v2): seed property_values.issue_description from Step 1 description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related bugs surfaced when walking the v2 builder without a real classifier (which currently 501s): 1. Step 1's free-text `description` never flowed into `property_values.issue_description`. Step 3's Refine screen rendered the "Original" column from `property_values`, so issue_description showed up empty even though the user had typed it in Step 1. The missing required field then silently blocked the form's Continue button (RefineSpec validator). 2. `refine_submitting` was set true by FormContainer on submit but never reset by the v2 handler. After a successful click the flag stayed true; if the user navigated back to Step 3 the button would be permanently disabled. For (1): seed `property_values.issue_description = description` at the start of classify_then_continue, before the classifier call. Real classifier response (when it ships) still overwrites; on 501 fallback we keep the user's input as the issue description. For (2): clear `refine_submitting` before advancing — the handler doesn't await any network call so there's no submitting state to preserve. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../specs_v2/[project_id]/[task_id]/+page.svelte | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte index 7f0fe215e4..95e5439ca6 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte @@ -110,6 +110,17 @@ classifying = true classify_error = null try { + // Seed property_values.issue_description from the free-text description + // up front. This is the fallback shape for the "issue" default — when + // the classifier ships, it'll overwrite below. Done here so Step 3's + // Refine "Original" column reflects what the user typed in Step 1 + // (and Step 2's refine_spec_with_question_answers has something to + // refine from), even if classification fails. + property_values = { + ...property_values, + issue_description: description, + } + const { data, error } = await client.POST( "/api/copilot/classify_spec_description", { @@ -253,6 +264,11 @@ // re-analyze, it just uses whatever refined_property_values the user // finalized. function on_refine_submit() { + // FormContainer flips submitting=true on every submit and leaves it to + // the caller to reset. We dispatch the form forward immediately (no + // network call here), so clear the flag before advancing — otherwise + // RefineSpec's button stays disabled if the user navigates back. + refine_submitting = false on_advance_to_generate() } From b4b72e189bb1258891b0e9af2c4efbd3864842ef Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Fri, 5 Jun 2026 01:51:52 -0700 Subject: [PATCH 21/36] fix(specs-v2): fall back to first run config when task has no default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tasks created via the new-task wizard don't auto-set default_run_config_id, so first-time multi-turn users were blocked at Step 4 with "Task has no default run config — set one in task settings" even though the task had a perfectly usable run config available. Prefer the default when set; otherwise pick the first available config. Only error when the task has zero configs (genuinely unrunnable). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../[project_id]/[task_id]/+page.svelte | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte index 95e5439ca6..a3179766d6 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte @@ -396,10 +396,12 @@ real_multi_turn_batch_tag = null try { - // 1. Resolve target_run_config from the task's default run config. - if (!task?.id || !task.default_run_config_id) { - generation_error = - "Task has no default run config — set one in task settings." + // 1. Resolve target_run_config: prefer the task's default; if none + // set, fall back to the first available run config so the user + // doesn't have to detour into task settings just to try v2. Only + // error when the task has zero configs (genuinely unrunnable). + if (!task?.id) { + generation_error = "Task not loaded." return } await load_task_run_configs(project_id, task.id) @@ -407,17 +409,18 @@ get(run_configs_by_task_composite_id)[ get_task_composite_id(project_id, task.id) ] ?? [] - const default_config = run_configs.find( - (c) => c.id === task!.default_run_config_id, - ) - if (!default_config) { - generation_error = "Task default run config not found." + if (run_configs.length === 0) { + generation_error = + "Task has no run configs — create one before running multi-turn." return } - const rcp = default_config.run_config_properties + const chosen_config = + run_configs.find((c) => c.id === task!.default_run_config_id) ?? + run_configs[0] + const rcp = chosen_config.run_config_properties if (!isKilnAgentRunConfig(rcp)) { generation_error = - "Multi-turn requires a Kiln Agent run config on the task default." + "Multi-turn requires a Kiln Agent run config; the selected one isn't." return } const target_run_config = { From 04c9b5c47cafa25b855c8fbede88afe839a1cdf7 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Fri, 5 Jun 2026 01:57:42 -0700 Subject: [PATCH 22/36] feat(specs-v2): simpler Step 3 Refine UI for multi-turn evals v1's RefineSpec component (which v2 was reusing for both modes) is overkill for multi-turn: examples fields don't apply (the synthetic- user chains from Step 4 are the real "examples"), and the Original / Refined two-column diff plus duplicate name inputs add UX friction when only the description is meaningfully editable. For multi-turn tasks, render a stripped-down variant: one editable name input, one editable description textarea (with the refinement reason shown inline when present), and unincorporated-feedback note if any. Single-turn keeps using v1's RefineSpec. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../[project_id]/[task_id]/+page.svelte | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte index a3179766d6..e5eda2460f 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte @@ -862,13 +862,71 @@ {/if} {:else if current_step === "refine"} - + {#if refined_preview_loading}
    Refining your eval…
    + {:else if is_multi_turn} + +

    Refine your eval

    +

    + Review the refined description below. The synthetic-user run in Step 4 + will probe your agent against this spec. +

    + +
    + +
    + +
    + + {#if suggested_edits.issue_description?.reason_for_edit} +
    + Refinement: {suggested_edits.issue_description.reason_for_edit} +
    + {/if} +
    + + {#if not_incorporated_feedback} +
    + Unincorporated feedback: + {not_incorporated_feedback} +
    + {/if} + +
    + + +
    {:else} + Date: Fri, 5 Jun 2026 02:10:58 -0700 Subject: [PATCH 23/36] feat(specs-v2): distinct Step 4 phase indicator for generate_cases call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 4 multi-turn does two sequential operations: first a single LLM call to /multiturn_sdg/generate_cases (can take 5-15s for 10 cases), then the longer streaming run_cases_batch SSE batch (minutes). The UI was showing the same "0 of 10 ready" message during the case-gen call, which is misleading — no chains are running yet, the copilot is still authoring the personas. Add a multi_turn_phase state ("idle" | "generating_cases" | "running_batch") and branch the Step 4 copy on it. Title + subtitle + status line all reflect what's actually happening behind the spinner. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../[project_id]/[task_id]/+page.svelte | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte index e5eda2460f..86af28fb9f 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte @@ -294,6 +294,11 @@ let multi_turn_chains: Chain[] = [] let multi_turn_progress = 0 // 0..N as cases complete const multi_turn_total = NUM_CASES + // Sub-phase for the Step 4 UI: distinguishes the up-front LLM call + // (generate_cases) from the longer batch-run stream so the user sees + // distinct progress instead of one ambiguous spinner. + type MultiTurnPhase = "idle" | "generating_cases" | "running_batch" + let multi_turn_phase: MultiTurnPhase = "idle" // Real batch_tag from run_cases_batch's BatchStartedEvent. The on_save // multi-turn branch uses this to tell the backend which chains to tag // for the eval dataset. @@ -394,6 +399,7 @@ multi_turn_progress = 0 multi_turn_chains = [] real_multi_turn_batch_tag = null + multi_turn_phase = "idle" try { // 1. Resolve target_run_config: prefer the task's default; if none @@ -430,6 +436,7 @@ } // 2. Generate synthetic-user cases via copilot. + multi_turn_phase = "generating_cases" const refined_description = (refined_property_values.issue_description as string | null) ?? description @@ -451,6 +458,7 @@ // 3. Stream run_cases_batch. The endpoint is POST so we can't use // EventSource (which is GET-only). Manual fetch + ReadableStream + // SSE line parsing — same pattern as streaming_chat.ts. + multi_turn_phase = "running_batch" const url = `${base_url}/api/projects/${project_id}/tasks/${task_id}/multiturn_sdg/run_cases_batch` const response = await fetch(url, { method: "POST", @@ -951,14 +959,19 @@ {:else if current_step === "generate"}

    - {#if is_multi_turn} + {#if is_multi_turn && multi_turn_phase === "generating_cases"} + Generating {NUM_CASES} synthetic-user cases + {:else if is_multi_turn} Running synthetic conversations {:else} Generating examples {/if}

    - {#if is_multi_turn} + {#if is_multi_turn && multi_turn_phase === "generating_cases"} + Asking the copilot to author {NUM_CASES} persona-driven scenarios that + probe your spec. + {:else if is_multi_turn} Driving {multi_turn_total} cases against your agent. This will take a moment. {:else} Generating ~10 sample inputs and outputs based on your spec. @@ -968,7 +981,11 @@ {#if generation_loading}

    - {#if is_multi_turn} + {#if is_multi_turn && multi_turn_phase === "generating_cases"} +
    + Generating {NUM_CASES} cases… +
    + {:else if is_multi_turn}
    {multi_turn_progress} of {multi_turn_total} ready
    From f35d2fc65e0e518c76a0a813030b1522c4475d15 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Fri, 5 Jun 2026 02:15:35 -0700 Subject: [PATCH 24/36] fix(specs-v2): disable Save until all multi-turn chains are reviewed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 5 multi-turn was letting users hit Save with chain_verdicts still null — the golden ratings ended up empty and the underlying eval got created with no validated examples. Disable the button until every chain has been marked pass or fail; add a tooltip on the disabled state so the user understands what's blocking. Single-turn already enforces this via v1's ReviewExamples component (its submit_disabled = !all_feedback_aligned && !all_examples_reviewed) so no change needed there. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../[project_id]/[task_id]/+page.svelte | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte index 86af28fb9f..e7e8bfef3c 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte @@ -617,6 +617,12 @@ chain_verdicts.length !== multi_turn_chains.length ? multi_turn_chains.map(() => ({ verdict: null, feedback: "" })) : chain_verdicts + // True only when every multi-turn chain has been marked pass or fail. + // Save button is disabled until then so the golden ratings are complete. + $: all_chains_reviewed = + multi_turn_chains.length > 0 && + chain_verdicts.length === multi_turn_chains.length && + chain_verdicts.every((v) => v.verdict !== null) // ── Step 6 state — save let saving = false @@ -1095,9 +1101,20 @@ class="btn btn-ghost" on:click={() => (current_step = "generate")}>← Back - +
    + +
    {:else}
    Step @@ -781,23 +835,15 @@
    {task_error}
    {:else if current_step === "describe"} -

    What should this eval check?

    -

    - Describe in plain language. We'll structure it for you. -

    - -
    - Spec name (auto-derived after Continue) -
    - {#if classify_error}
    {classify_error}
    {/if} @@ -856,19 +902,14 @@ {:else if is_multi_turn} -

    Refine your eval

    -

    - Review the refined description below. The synthetic-user run in Step 4 - will probe your agent against this spec. -

    -
    @@ -935,26 +976,6 @@ {/if} {:else if current_step === "generate"} -

    - {#if is_multi_turn && multi_turn_phase === "generating_cases"} - Generating {NUM_CASES} synthetic-user cases - {:else if is_multi_turn} - Running synthetic conversations - {:else} - Generating examples - {/if} -

    -

    - {#if is_multi_turn && multi_turn_phase === "generating_cases"} - Asking the copilot to author {NUM_CASES} persona-driven scenarios that - probe your spec. - {:else if is_multi_turn} - Driving {multi_turn_total} cases against your agent. This will take a moment. - {:else} - Generating ~10 sample inputs and outputs based on your spec. - {/if} -

    - {#if generation_loading}
    @@ -994,10 +1015,6 @@ {:else if current_step === "review"} {#if is_multi_turn} -

    Review

    -

    - Read each conversation. Mark each chain Pass or Fail. -

    @@ -1010,9 +1027,6 @@
    Conversation {ci + 1} of {multi_turn_chains.length}
    -
    - {chain.persona_summary} -
    • - +
  • - Evals V2 -
    Beta
    + Evals Legacy
    • diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte index 9c3bf8916e..9b180e480f 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte @@ -724,10 +724,17 @@ } async function check_kiln_copilot_and_proceed() { + posthog.capture("eval_v2_cta_clicked", { + branch: has_kiln_copilot ? "v2" : "v1_manual", + has_pro: has_kiln_copilot, + }) if (!has_kiln_copilot) { goto(`/specs/${project_id}/${task_id}/select_workflow`) } else { - goto(`/specs/${project_id}/${task_id}/select_template`) + // Pro users land on the v2 builder. The legacy template carousel + // remains reachable via the "Evals Legacy" sidebar entry during + // the bug bash; remove the fallback once v2 ships GA. + goto(`/specs/${project_id}/${task_id}/builder`) } } diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte index 0aca68fc56..c7f68a18d2 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte @@ -471,6 +471,13 @@ ]} action_buttons={action_buttons(evaluator)} > + {#if evaluator?.evaluation_data_type === "full_trace"} +
    + Multi-turn run-configuration comparison is coming soon. Single-model + scoring works today via "Run Eval"; the table below assumes single-turn + re-drive and won't produce meaningful results for multi-turn evals. +
    + {/if} {#if loading}
    diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte similarity index 65% rename from app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte rename to app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte index 8a665a9e71..5425247128 100644 --- a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte @@ -1,5 +1,5 @@ -
    - -
    - Step - {STEP_INDEX[current_step]} - of {TOTAL_STEPS} - {#if is_multi_turn} - multi-turn - {/if} + {#if $kilnCopilotConnected === null} +
    +
    - - {#if task_loading} -
    Loading task…
    - {:else if task_error} -
    {task_error}
    - {:else if current_step === "describe"} - - - - {#if classify_error} -
    {classify_error}
    - {/if} - -
    - - + {:else if $kilnCopilotConnected === false} + + {:else} +
    + +
    + Step + {STEP_INDEX[current_step]} + of {TOTAL_STEPS} + {#if is_multi_turn} + multi-turn + {/if}
    - {:else if current_step === "clarify"} - - {#if questions_loading} -
    - -
    Generating clarifying questions…
    -
    - {:else if questions_error} -
    {questions_error}
    - {:else if question_set} - Loading task…
    + {:else if task_error} +
    {task_error}
    + {:else if current_step === "describe"} + + -
    - -
    - {/if} - {:else if current_step === "refine"} - - {#if refined_preview_loading} -
    - -
    Refining your eval…
    -
    - {:else if is_multi_turn} - -
    - -
    -
    - - {#if suggested_edits.issue_description?.reason_for_edit} -
    - Refinement: {suggested_edits.issue_description.reason_for_edit} -
    - {/if} -
    +

    + Beta: every spec is treated as an Issue eval. Other spec types are + coming soon. +

    - {#if not_incorporated_feedback} -
    - Unincorporated feedback: - {not_incorporated_feedback} -
    + {#if classify_error} +
    {classify_error}
    {/if}
    - ← Cancel
    - {:else} - - -
    - -
    - {/if} - {:else if current_step === "generate"} - - {#if generation_loading} -
    - - {#if is_multi_turn && multi_turn_phase === "generating_cases"} -
    - Generating {NUM_CASES} cases… -
    - {:else if is_multi_turn} -
    - {multi_turn_progress} of {multi_turn_total} ready + {:else if current_step === "clarify"} + + {#if questions_loading} +
    + +
    Generating clarifying questions…
    +
    + {:else if questions_error} +
    {questions_error}
    + {:else if question_set} + +
    + +
    + {/if} + {:else if current_step === "refine"} + + {#if refined_preview_loading} +
    + +
    Refining your eval…
    +
    + {:else if is_multi_turn} + +
    + +
    + +
    + + {#if suggested_edits.issue_description?.reason_for_edit} +
    + Refinement: {suggested_edits.issue_description.reason_for_edit} +
    + {/if} +
    + + {#if not_incorporated_feedback} +
    + Unincorporated feedback: + {not_incorporated_feedback}
    - {:else} -
    Generating examples…
    {/if} -
    - {/if} - {#if generation_error} -
    {generation_error}
    -
    +
    + + +
    + {:else} + + +
    + +
    + {/if} + {:else if current_step === "generate"} + + {#if is_multi_turn && multi_turn_fallback_run_config_name} +
    + Using run config {multi_turn_fallback_run_config_name} — set a default in task settings to silence this notice. +
    + {/if} + {#if generation_loading} +
    + + {#if is_multi_turn && multi_turn_phase === "generating_cases"} +
    + Generating {NUM_CASES} cases… +
    + {:else if is_multi_turn} +
    + {multi_turn_progress} of {multi_turn_total} ready +
    + {:else} +
    Generating examples…
    + {/if} +
    + {/if} + + {#if generation_error} +
    {generation_error}
    +
    + +
    + {/if} + +
    - Retry → -
    - {/if} - -
    - -
    - {:else if current_step === "review"} - - {#if is_multi_turn} - + {#if is_multi_turn} + -
    - {#each multi_turn_chains as chain, ci} -
    -
    -
    -
    -
    - Conversation {ci + 1} of {multi_turn_chains.length} +
    + {#each multi_turn_chains as chain, ci} +
    +
    +
    +
    +
    + Conversation {ci + 1} of {multi_turn_chains.length} +
    +
    +
    + +
    -
    - - -
    -
    -
    - {#each chain.trace as turn} -
    -
    - {turn.role} +
    + {#each chain.trace as turn} +
    +
    + {turn.role} +
    +
    {turn.content}
    -
    {turn.content}
    -
    - {/each} -
    + {/each} +
    - {#if chain_verdicts[ci]?.verdict !== null} - - {/if} + {#if chain_verdicts[ci]?.verdict !== null} + + {/if} +
    -
    - {/each} -
    -
    - -
    + {/each} +
    +
    - Save → - +
    + +
    -
    - {:else} - - -
    + +
    + +
    + {/if} + {:else if current_step === "save"} + + {#if saving} +
    + +
    Saving…
    +
    + {:else if save_error} +
    {save_error}
    +
    + +
    + {/if} + +
    (current_step = "review")} + disabled={saving}>← Back
    - {/if} - {:else if current_step === "save"} - - {#if saving} + {:else if current_step === "done"} +
    - -
    Saving…
    -
    - {:else if save_error} -
    {save_error}
    -
    - +
    ✓
    +

    Spec saved

    +
    {/if} - -
    - -
    - {:else if current_step === "done"} - -
    -
    ✓
    -

    Spec saved

    - -
    - {/if} -
    +
    + {/if} diff --git a/app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.ts similarity index 100% rename from app/web_ui/src/routes/(app)/specs_v2/[project_id]/[task_id]/+page.ts rename to app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.ts From a572714c0fc03ca430fa3089a27016148bcc1e42 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Tue, 9 Jun 2026 01:03:29 -0700 Subject: [PATCH 28/36] =?UTF-8?q?fix(specs-v2):=20UI=20polish=20=E2=80=94?= =?UTF-8?q?=20width,=20alerts,=20abort=20plumbing,=20Step=201=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Match v1 spec_builder's width pattern: wrap the entire AppPage in the page_max_w div so title and body share the same constraint. - Replace raw alert/info divs with the shared Warning component (6 sites in builder + 1 in compare_run_configs). - Standardize Back/Cancel buttons to btn-ghost btn-sm. - Wire classify_error through FormElement's error_message prop. - Bump step indicator from text-xs to text-sm to match v1. - Drop the "Beta: every spec treated as Issue" hint — coordinate with Mike via the bug bash announcement instead. - Add v1's AbortController plumbing (abort_copilot_request, new_copilot_abort_signal, is_abort_error) and onDestroy cleanup. Long-running Copilot calls pass the signal, Back buttons cancel in-flight requests, catch blocks silently ignore AbortError. Co-Authored-By: Claude Opus 4.7 --- .../compare_run_configs/+page.svelte | 10 +- .../[task_id]/builder/+page.svelte | 751 ++++++++++-------- 2 files changed, 424 insertions(+), 337 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte index c7f68a18d2..c03815f737 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/[spec_id]/[eval_id]/compare_run_configs/+page.svelte @@ -472,11 +472,11 @@ action_buttons={action_buttons(evaluator)} > {#if evaluator?.evaluation_data_type === "full_trace"} -
    - Multi-turn run-configuration comparison is coming soon. Single-model - scoring works today via "Run Eval"; the table below assumes single-turn - re-drive and won't produce meaningful results for multi-turn evals. -
    + {/if} {#if loading}
    diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte index 5425247128..631f62ccd0 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte @@ -1,7 +1,7 @@ - - {#if $kilnCopilotConnected === null} -
    - -
    - {:else if $kilnCopilotConnected === false} - - {:else} -
    - -
    - Step - {STEP_INDEX[current_step]} - of {TOTAL_STEPS} - {#if is_multi_turn} - multi-turn - {/if} + +
    + + {#if $kilnCopilotConnected === null} +
    +
    - - {#if task_loading} -
    Loading task…
    - {:else if task_error} -
    {task_error}
    - {:else if current_step === "describe"} - - - -

    - Beta: every spec is treated as an Issue eval. Other spec types are - coming soon. -

    - - {#if classify_error} -
    {classify_error}
    - {/if} - -
    - - + {:else if $kilnCopilotConnected === false} + + {:else} +
    + +
    + Step + {STEP_INDEX[current_step]} + of {TOTAL_STEPS} + {#if is_multi_turn} + multi-turn + {/if}
    - {:else if current_step === "clarify"} - - {#if questions_loading} -
    - -
    Generating clarifying questions…
    -
    - {:else if questions_error} -
    {questions_error}
    - {:else if question_set} - Loading task…
    + {:else if task_error} + + {:else if current_step === "describe"} + + -
    + +
    + + {#if classifying} + + Classifying… + {:else} + Continue → + {/if} +
    - {/if} - {:else if current_step === "refine"} - - {#if refined_preview_loading} -
    - -
    Refining your eval…
    -
    - {:else if is_multi_turn} - -
    - + {#if questions_loading} +
    + +
    Generating clarifying questions…
    +
    +
    + +
    + {:else if questions_error} + +
    + +
    + {:else if question_set} + -
    +
    + +
    + {/if} + {:else if current_step === "refine"} + + {#if refined_preview_loading} +
    + +
    Refining your eval…
    +
    +
    + +
    + {:else if is_multi_turn} + +
    + +
    -
    - - {#if suggested_edits.issue_description?.reason_for_edit} -
    - Refinement: {suggested_edits.issue_description.reason_for_edit} -
    +
    + + {#if suggested_edits.issue_description?.reason_for_edit} +
    + Refinement: {suggested_edits.issue_description + .reason_for_edit} +
    + {/if} +
    + + {#if not_incorporated_feedback} + {/if} -
    - {#if not_incorporated_feedback} -
    - Unincorporated feedback: +
    + + +
    + {:else} + + +
    + +
    + {/if} + {:else if current_step === "generate"} + + {#if is_multi_turn && multi_turn_fallback_run_config_name} + + {/if} + {#if generation_loading} +
    + + {#if is_multi_turn && multi_turn_phase === "generating_cases"} +
    + Generating {NUM_CASES} cases… +
    + {:else if is_multi_turn} +
    + {multi_turn_progress} of {multi_turn_total} ready +
    + {:else} +
    + Generating examples… +
    + {/if} +
    + {/if} + + {#if generation_error} + +
    +
    {/if}
    - - Generate conversations → -
    - {:else} - - -
    - -
    - {/if} - {:else if current_step === "generate"} - - {#if is_multi_turn && multi_turn_fallback_run_config_name} -
    - Using run config {multi_turn_fallback_run_config_name} — set a default in task settings to silence this notice. -
    - {/if} - {#if generation_loading} -
    - - {#if is_multi_turn && multi_turn_phase === "generating_cases"} -
    - Generating {NUM_CASES} cases… -
    - {:else if is_multi_turn} -
    - {multi_turn_progress} of {multi_turn_total} ready -
    - {:else} -
    Generating examples…
    - {/if} -
    - {/if} - - {#if generation_error} -
    {generation_error}
    -
    - -
    - {/if} - -
    - -
    - {:else if current_step === "review"} - - {#if is_multi_turn} - + {#if is_multi_turn} + -
    - {#each multi_turn_chains as chain, ci} -
    -
    -
    -
    -
    - Conversation {ci + 1} of {multi_turn_chains.length} +
    + {#each multi_turn_chains as chain, ci} +
    +
    +
    +
    +
    + Conversation {ci + 1} of {multi_turn_chains.length} +
    +
    +
    + +
    -
    - - -
    -
    -
    - {#each chain.trace as turn} -
    -
    - {turn.role} +
    + {#each chain.trace as turn} +
    +
    + {turn.role} +
    +
    {turn.content}
    -
    {turn.content}
    -
    - {/each} -
    + {/each} +
    - {#if chain_verdicts[ci]?.verdict !== null} - - {/if} + {#if chain_verdicts[ci]?.verdict !== null} + + {/if} +
    -
    - {/each} -
    -
    - -
    + {/each} +
    +
    - Save → - +
    + +
    -
    - {:else} - - -
    + +
    + +
    + {/if} + {:else if current_step === "save"} + + {#if saving} +
    + +
    Saving…
    +
    + {:else if save_error} + +
    + +
    + {/if} + +
    (current_step = "review")} + disabled={saving}>← Back
    - {/if} - {:else if current_step === "save"} - - {#if saving} + {:else if current_step === "done"} +
    - -
    Saving…
    -
    - {:else if save_error} -
    {save_error}
    -
    - +
    ✓
    +

    Spec saved

    +
    {/if} - -
    - -
    - {:else if current_step === "done"} - -
    -
    ✓
    -

    Spec saved

    - -
    - {/if} -
    - {/if} - +
    + {/if} + +
    From 33c9ba8753108f3fcbb6203269108c75e68e4ca8 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Tue, 9 Jun 2026 02:31:05 -0700 Subject: [PATCH 29/36] feat(specs-v2): paginated multi-turn review UI Replace Step 5's vertical card stack with a focused paginator: one conversation at a time, dot-row navigator at the top, persona + verdict buttons inside the card, single bottom action bar. - Pass auto-advances to the next un-reviewed conversation; Fail keeps the user on the current case and focuses the feedback input, since the judge prompt needs the reason to learn from. - Save is gated until every chain has a verdict AND every failed chain has a non-empty reason. Tooltip explains what's missing. - Single bottom action bar owns Back/Prev/Next/Save so the user sees one set of buttons instead of two stacked rows. - Dots are small solid markers with numbered labels below, connected by solid lines (matches DaisyUI .steps line convention). - Feedback wording mirrors v1 review_examples: "Describe why this fails" / "Describe why this passes (optional)". Adds a DEV-ONLY mock route at /dev_mock_review for iterating on the paginator without running through the full eval-builder flow. TODO-marked for removal post-bash. Co-Authored-By: Claude Opus 4.7 --- .../routes/(app)/dev_mock_review/+page.svelte | 348 ++++++++++++++++++ .../src/routes/(app)/dev_mock_review/+page.ts | 1 + .../[task_id]/builder/+page.svelte | 116 ++---- .../multi_turn_review_paginator.svelte | 208 +++++++++++ 4 files changed, 579 insertions(+), 94 deletions(-) create mode 100644 app/web_ui/src/routes/(app)/dev_mock_review/+page.svelte create mode 100644 app/web_ui/src/routes/(app)/dev_mock_review/+page.ts create mode 100644 app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/multi_turn_review_paginator.svelte diff --git a/app/web_ui/src/routes/(app)/dev_mock_review/+page.svelte b/app/web_ui/src/routes/(app)/dev_mock_review/+page.svelte new file mode 100644 index 0000000000..ebfe55a7a8 --- /dev/null +++ b/app/web_ui/src/routes/(app)/dev_mock_review/+page.svelte @@ -0,0 +1,348 @@ + + + + +
    + +
    + +
    + Step + 5 + of 6 + multi-turn + + mock + +
    + + alert("Back to Generate (mock)")} + on_save={() => alert("Save → (mock)")} + save_disabled={!all_reviewed} + save_disabled_tooltip={all_reviewed + ? null + : "Mark each conversation pass or fail; failed conversations need a reason."} + /> +
    +     +
    diff --git a/app/web_ui/src/routes/(app)/dev_mock_review/+page.ts b/app/web_ui/src/routes/(app)/dev_mock_review/+page.ts new file mode 100644 index 0000000000..9786e09d9a --- /dev/null +++ b/app/web_ui/src/routes/(app)/dev_mock_review/+page.ts @@ -0,0 +1 @@ +export const prerender = false diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte index 631f62ccd0..33ef7f37b2 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte @@ -18,6 +18,7 @@ import Questions from "../spec_builder/questions.svelte" import RefineSpec from "../spec_builder/refine_spec.svelte" import ReviewExamples from "../spec_builder/review_examples.svelte" + import MultiTurnReviewPaginator from "./multi_turn_review_paginator.svelte" import { spec_field_configs } from "../select_template/spec_templates" import type { SuggestedEdit, ReviewRow } from "../spec_utils" import { KilnError } from "$lib/utils/error_handlers" @@ -678,12 +679,17 @@ chain_verdicts.length !== multi_turn_chains.length ? multi_turn_chains.map(() => ({ verdict: null, feedback: "" })) : chain_verdicts - // True only when every multi-turn chain has been marked pass or fail. - // Save button is disabled until then so the golden ratings are complete. + // Save is disabled until every chain has a verdict AND every fail- + // verdict has a non-empty reason. Pass-without-feedback is fine — the + // judge prompt only needs the WHY when something went wrong. $: all_chains_reviewed = multi_turn_chains.length > 0 && chain_verdicts.length === multi_turn_chains.length && - chain_verdicts.every((v) => v.verdict !== null) + chain_verdicts.every( + (v) => + v.verdict !== null && + (v.verdict !== "fail" || v.feedback.trim().length > 0), + ) // ── Step 6 state — save let saving = false @@ -1159,97 +1165,19 @@ {:else if current_step === "review"} {#if is_multi_turn} - -
    - {#each multi_turn_chains as chain, ci} -
    -
    -
    -
    -
    - Conversation {ci + 1} of {multi_turn_chains.length} -
    -
    -
    - - -
    -
    - -
    - {#each chain.trace as turn} -
    -
    - {turn.role} -
    -
    {turn.content}
    -
    - {/each} -
    - - {#if chain_verdicts[ci]?.verdict !== null} - - {/if} -
    -
    - {/each} -
    -
    - -
    - -
    -
    + { + abort_copilot_request() + current_step = "generate" + }} + on_save={on_advance_to_save} + save_disabled={!all_chains_reviewed} + save_disabled_tooltip={all_chains_reviewed + ? null + : "Mark each conversation pass or fail; failed conversations need a reason."} + /> {:else} +
    + {#each chains as _, i} + {@const v = verdicts[i]} + {@const fail_missing_reason = v?.verdict === "fail" && !v.feedback.trim()} + {#if i > 0} +
    + {/if} +
    + + + {i + 1} + +
    + {/each} +
    + + + {#if current_chain} +
    +
    +
    +
    + {#if current_chain.persona_summary} + Persona: {current_chain.persona_summary} + {/if} +
    +
    + + +
    +
    +
    + {#each current_chain.trace as turn} +
    +
    {turn.role}
    +
    {turn.content}
    +
    + {/each} +
    + + + {#if current_verdict?.verdict !== null} + +

    + {current_verdict?.verdict === "fail" + ? "Required. Detailed explanations will improve the judge." + : "Detailed explanations will improve the judge."} +

    + {/if} +
    +
    + {/if} + + +
    + +
    + + +
    +
    + +
    +
    +
    From c8449e057f08bd1c336cfe4730a42e3418d2700e Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Wed, 24 Jun 2026 11:13:26 -0700 Subject: [PATCH 30/36] chore(agent_checks): add policy annotations for multiturn_sdg endpoints Backfill the checked-in agent policy annotation files for the two new multiturn_sdg routes (generate_cases, run_cases_batch) so the API bindings CI check passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ct_id_tasks_task_id_multiturn_sdg_generate_cases.json | 9 +++++++++ ...t_id_tasks_task_id_multiturn_sdg_run_cases_batch.json | 9 +++++++++ 2 files changed, 18 insertions(+) create mode 100644 libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_generate_cases.json create mode 100644 libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_run_cases_batch.json diff --git a/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_generate_cases.json b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_generate_cases.json new file mode 100644 index 0000000000..80ec142869 --- /dev/null +++ b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_generate_cases.json @@ -0,0 +1,9 @@ +{ + "method": "post", + "path": "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/generate_cases", + "agent_policy": { + "permission": "allow", + "requires_approval": true, + "approval_description": "Generate synthetic-user cases? Uses an LLM call (cost)." + } +} diff --git a/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_run_cases_batch.json b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_run_cases_batch.json new file mode 100644 index 0000000000..a30a3e5c13 --- /dev/null +++ b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_tasks_task_id_multiturn_sdg_run_cases_batch.json @@ -0,0 +1,9 @@ +{ + "method": "post", + "path": "/api/projects/{project_id}/tasks/{task_id}/multiturn_sdg/run_cases_batch", + "agent_policy": { + "permission": "allow", + "requires_approval": true, + "approval_description": "Run a multi-turn synthetic-user batch? Invokes the target model and the SU driver model for several turns per case (cost)." + } +} From 384fb3690e2b7539c8928a18bd587511efca2f7a Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:50:58 -0700 Subject: [PATCH 31/36] fix(api_client): drop hardcoded multipart Content-Type from vendored SDK The SDK re-vendor reintroduced a hardcoded "Content-Type: multipart/form-data; boundary=+++" on the prompt optimization endpoint, an openapi-python-client multipart bug main had already fixed. The placeholder boundary prevents httpx from negotiating its own, corrupting the multipart body. Remove the line to match main and satisfy test_multipart_boundary. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...timization_job_v1_jobs_prompt_optimization_job_start_post.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py index ba1ec9502d..8b282ed871 100644 --- a/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py +++ b/app/desktop/studio_server/api_client/kiln_ai_server_client/api/jobs/start_prompt_optimization_job_v1_jobs_prompt_optimization_job_start_post.py @@ -26,8 +26,6 @@ def _get_kwargs( _kwargs["files"] = body.to_multipart() - headers["Content-Type"] = "multipart/form-data; boundary=+++" - _kwargs["headers"] = headers return _kwargs From 5e1d493b81bb095e22de4b3dee55fcb1120fe296 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:17:30 -0700 Subject: [PATCH 32/36] chore(agent_checks): add policy annotation for copilot classify_spec_description Combining branches pulled in the new /api/copilot/classify_spec_description endpoint; add its checked-in agent policy annotation so the API bindings CI check passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../post_api_copilot_classify_spec_description.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 libs/server/kiln_server/utils/agent_checks/annotations/post_api_copilot_classify_spec_description.json diff --git a/libs/server/kiln_server/utils/agent_checks/annotations/post_api_copilot_classify_spec_description.json b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_copilot_classify_spec_description.json new file mode 100644 index 0000000000..06f65c4e84 --- /dev/null +++ b/libs/server/kiln_server/utils/agent_checks/annotations/post_api_copilot_classify_spec_description.json @@ -0,0 +1,9 @@ +{ + "method": "post", + "path": "/api/copilot/classify_spec_description", + "agent_policy": { + "permission": "allow", + "requires_approval": true, + "approval_description": "Classify a free-text spec description?" + } +} From 99774ed61c209f325b7e54802af46eba9fa2ea38 Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Thu, 25 Jun 2026 01:31:35 -0700 Subject: [PATCH 33/36] fix(specs): unblock multi-turn tasks from reaching the v2 builder The v1 listing page had pre-v2 guards that hid the "Create Eval" CTA and replaced the body with "Evals are not supported for multi-turn tasks." when task.turn_mode === "multiturn". Those guards were defensive code from when v1's spec_builder couldn't handle multi-turn; v2's builder is multi-turn-aware so they no longer apply. Drop the guards and the now-dead task-load plumbing they fed (is_multiturn reactive, task / task_loading state, load_task_for_page, and the Task / load_task / Warning imports). Co-Authored-By: Claude Opus 4.7 --- .../specs/[project_id]/[task_id]/+page.svelte | 33 ++----------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte index 457958e3c7..d9ccebad32 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/+page.svelte @@ -5,9 +5,7 @@ import { client } from "$lib/api_client" import { onMount } from "svelte" import Intro from "$lib/ui/intro.svelte" - import type { Spec, SpecStatus, Eval, Priority, Task } from "$lib/types" - import { load_task } from "$lib/stores" - import Warning from "$lib/ui/warning.svelte" + import type { Spec, SpecStatus, Eval, Priority } from "$lib/types" import { goto, replaceState } from "$app/navigation" import Dialog from "$lib/ui/dialog.svelte" import FilterTagsDialog from "$lib/ui/filter_tags_dialog.svelte" @@ -48,12 +46,7 @@ let settings_error: KilnError | null = null let has_kiln_copilot = false - let task: Task | null = null - let task_loading = true - $: is_multiturn = task?.turn_mode === "multiturn" - - $: loading = - specs_loading || evals_loading || settings_loading || task_loading + $: loading = specs_loading || evals_loading || settings_loading $: error = specs_error || evals_error || settings_error type TableRow = @@ -149,18 +142,6 @@ $: if (project_id && task_id) { load_specs(project_id, task_id) load_evals(project_id, task_id) - load_task_for_page(project_id, task_id) - } - - async function load_task_for_page( - req_project_id: string, - req_task_id: string, - ) { - task_loading = true - const loaded = await load_task(req_project_id, req_task_id) - if (req_project_id !== project_id || req_task_id !== task_id) return - task = loaded - task_loading = false } onMount(async () => { @@ -764,7 +745,7 @@ subtitle="Define the behaviours to enforce or avoid for your task, and automatically measure quality." sub_subtitle={"Read the Docs"} sub_subtitle_link="https://docs.kiln.tech/docs/evals-and-specs" - action_buttons={is_empty || is_multiturn + action_buttons={is_empty ? [] : [ { @@ -781,14 +762,6 @@
    - {:else if is_multiturn} -
    - -
    {:else if error}
    {error?.getMessage() || "An unknown error occurred"} From 7e02c9a542dd52127ad36dfb27beb7dfe39c7e8a Mon Sep 17 00:00:00 2001 From: aiyakitori <16633065+chiang-daniel@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:47:50 -0700 Subject: [PATCH 34/36] =?UTF-8?q?feat(specs-v2):=20builder=20polish=20?= =?UTF-8?q?=E2=80=94=20v1=20animations,=20manual-create=20link,=20Cmd-Ente?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bucket D of Steve's eval-builder bug-bash feedback: - Reuse v1 Analyzing/Refining/Questioning/Saving animations on Steps 2-4 and 6 instead of bare dot-spinners; multi-turn keeps its live progress count via a reactive caption. - Add a 'Create manually' link under Step 1 to the legacy template flow. - Cmd/Ctrl-Enter on the bespoke primary buttons (Step 1, multi-turn refine, multi-turn save); FormContainer-backed steps already had it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../[task_id]/builder/+page.svelte | 114 +++++++++++++----- 1 file changed, 86 insertions(+), 28 deletions(-) diff --git a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte index 33ef7f37b2..8d7c26bbeb 100644 --- a/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte +++ b/app/web_ui/src/routes/(app)/specs/[project_id]/[task_id]/builder/+page.svelte @@ -19,6 +19,12 @@ import RefineSpec from "../spec_builder/refine_spec.svelte" import ReviewExamples from "../spec_builder/review_examples.svelte" import MultiTurnReviewPaginator from "./multi_turn_review_paginator.svelte" + // Reuse v1's themed loading animations on the wizard's transition screens + // instead of bare dot-spinners, so the two builders feel consistent. + import QuestioningAnimation from "$lib/ui/animations/questioning_animation.svelte" + import RefiningAnimation from "$lib/ui/animations/refining_animation.svelte" + import AnalyzingAnimation from "$lib/ui/animations/analyzing_animation.svelte" + import SavingAnimation from "$lib/ui/animations/saving_animation.svelte" import { spec_field_configs } from "../select_template/spec_templates" import type { SuggestedEdit, ReviewRow } from "../spec_utils" import { KilnError } from "$lib/utils/error_handlers" @@ -838,6 +844,43 @@ goto(`/specs/${project_id}/${task_id}`) } + // Escape hatch from Step 1 to the legacy manual builder (template carousel), + // for users who'd rather author the eval themselves than use the assistant. + function create_manually() { + posthog.capture("eval_v2_create_manually_clicked") + goto(`/specs/${project_id}/${task_id}/select_template`) + } + + // Cmd/Ctrl-Enter fires the current step's primary action — but only the + // steps with bespoke buttons. FormContainer-backed steps (clarify, single- + // turn refine/review) already handle it; skipping them avoids double-firing. + function handle_global_keydown(event: KeyboardEvent) { + if (!((event.metaKey || event.ctrlKey) && event.key === "Enter")) return + if (current_step === "describe") { + if (description.trim() && !classifying) { + event.preventDefault() + classify_then_continue() + } + } else if ( + current_step === "refine" && + is_multi_turn && + !refined_preview_loading + ) { + if ( + name.trim() && + (refined_property_values.issue_description ?? "").trim() + ) { + event.preventDefault() + on_refine_submit() + } + } else if (current_step === "review" && is_multi_turn) { + if (all_chains_reviewed) { + event.preventDefault() + on_advance_to_save() + } + } + } + // Auto-load questions when entering Step 2 $: if (current_step === "clarify" && !question_set && !questions_loading) { load_questions() @@ -899,8 +942,24 @@ $: page_title = page_title_for(current_step) $: page_subtitle = page_subtitle_for(current_step) $: page_max_w = page_max_w_for(current_step) + + // Step 4 animation caption. Multi-turn routes its live "X of N ready" count + // through here since the shared animation has no progress bar of its own. + $: generate_animation_description = is_multi_turn + ? multi_turn_phase === "generating_cases" + ? `Generating ${NUM_CASES} synthetic-user cases…` + : `Driving conversations — ${multi_turn_progress} of ${multi_turn_total} ready.` + : "Kiln is generating example data to review and creating a judge. Hold tight!" + + // Multi-turn save tags existing chains rather than generating a dataset, so + // the save copy differs from single-turn's generate-then-save. + $: save_animation_description = is_multi_turn + ? "Kiln is saving your eval and tagging the generated conversations. Hold tight!" + : "Kiln is generating test and training data for your eval before saving. Hold tight!" + + @@ -964,13 +1023,21 @@ {/if}
    + +
    + Prefer to set it up yourself? + +
    {:else if current_step === "clarify"} {#if questions_loading} -
    - -
    Generating clarifying questions…
    -
    +
    -
    {:else if is_multi_turn} @@ -1151,14 +1166,7 @@ /> {/if} -
    - +
    -
    {/if} {:else if current_step === "generate"} @@ -1237,25 +1236,38 @@
    {/if} -
    - -
    + {#if !generation_loading && !generation_error} +
    + {#if single_turn_examples.length > 0 || multi_turn_chains.length > 0} + + + {:else} + + + {/if} +
    + {/if} {:else if current_step === "review"} {#if is_multi_turn} { - abort_copilot_request() - current_step = "generate" - }} + on_back={() => history.back()} on_save={on_advance_to_save} save_disabled={!all_chains_reviewed} save_disabled_tooltip={all_chains_reviewed @@ -1279,15 +1291,6 @@ on:continue_to_refine={on_advance_to_save} on:create_spec_secondary={on_advance_to_save} /> -
    - -
    {/if} {:else if current_step === "save"} @@ -1303,14 +1306,6 @@ >
    {/if} - -
    - -
    {:else if current_step === "done"}