feat: Multi-turn eval

app/desktop/desktop_server.py

@@ -33,6 +33,7 @@
 from app.desktop.studio_server.eval_api import connect_evals_api
 from app.desktop.studio_server.finetune_api import connect_fine_tune_api
 from app.desktop.studio_server.import_api import connect_import_api
+from app.desktop.studio_server.multiturn_sdg_api import connect_multiturn_sdg_api
 from app.desktop.studio_server.prompt_api import connect_prompt_api
 from app.desktop.studio_server.prompt_optimization_job_api import (
     connect_prompt_optimization_job_api,
@@ -142,6 +143,7 @@ def make_app(tk_root: tk.Tk | None = None):
     connect_skill_api(app)
     connect_prompt_optimization_job_api(app)
     connect_copilot_api(app)
+    connect_multiturn_sdg_api(app)
     connect_git_sync_api(app)
     connect_agent_api(app)
     connect_dev_tools(app)

app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/__init__.py

@@ -0,0 +1 @@
+"""Contains endpoint functions for accessing the API"""

app/desktop/studio_server/api_client/kiln_ai_server_client/api/synthetic_user/generate_v1_synthetic_user_generate_post.py

@@ -0,0 +1,253 @@
+from http import HTTPStatus
+from typing import Any
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.generate_synthetic_users_request import GenerateSyntheticUsersRequest
+from ...models.generate_synthetic_users_response import GenerateSyntheticUsersResponse
+from ...models.generate_v1_synthetic_user_generate_post_response_401 import (
+    GenerateV1SyntheticUserGeneratePostResponse401,
+)
+from ...models.generate_v1_synthetic_user_generate_post_response_500 import (
+    GenerateV1SyntheticUserGeneratePostResponse500,
+)
+from ...models.generate_v1_synthetic_user_generate_post_response_502 import (
+    GenerateV1SyntheticUserGeneratePostResponse502,
+)
+from ...models.http_validation_error import HTTPValidationError
+from ...types import Response
+
+
+def _get_kwargs(
+    *,
+    body: GenerateSyntheticUsersRequest,
+) -> dict[str, Any]:
+    headers: dict[str, Any] = {}
+
+    _kwargs: dict[str, Any] = {
+        "method": "post",
+        "url": "/v1/synthetic_user/generate",
+    }
+
+    _kwargs["json"] = body.to_dict()
+
+    headers["Content-Type"] = "application/json"
+
+    _kwargs["headers"] = headers
+    return _kwargs
+
+
+def _parse_response(
+    *, client: AuthenticatedClient | Client, response: httpx.Response
+) -> (
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+    | None
+):
+    if response.status_code == 200:
+        response_200 = GenerateSyntheticUsersResponse.from_dict(response.json())
+
+        return response_200
+
+    if response.status_code == 401:
+        response_401 = GenerateV1SyntheticUserGeneratePostResponse401.from_dict(response.json())
+
+        return response_401
+
+    if response.status_code == 422:
+        response_422 = HTTPValidationError.from_dict(response.json())
+
+        return response_422
+
+    if response.status_code == 500:
+        response_500 = GenerateV1SyntheticUserGeneratePostResponse500.from_dict(response.json())
+
+        return response_500
+
+    if response.status_code == 502:
+        response_502 = GenerateV1SyntheticUserGeneratePostResponse502.from_dict(response.json())
+
+        return response_502
+
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: AuthenticatedClient | Client, response: httpx.Response
+) -> Response[
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+    body: GenerateSyntheticUsersRequest,
+) -> Response[
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+]:
+    """Generate
+
+     Return `num_cases` synthetic-user cases for the authoring UX.
+
+    Args:
+        body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate.
+
+            Generates `num_cases` synthetic-user cases designed to probe
+            `target_specification` against the agent described by `target_task_prompt`,
+            across multi-turn conversations.
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError]
+    """
+
+    kwargs = _get_kwargs(
+        body=body,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: AuthenticatedClient,
+    body: GenerateSyntheticUsersRequest,
+) -> (
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+    | None
+):
+    """Generate
+
+     Return `num_cases` synthetic-user cases for the authoring UX.
+
+    Args:
+        body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate.
+
+            Generates `num_cases` synthetic-user cases designed to probe
+            `target_specification` against the agent described by `target_task_prompt`,
+            across multi-turn conversations.
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError
+    """
+
+    return sync_detailed(
+        client=client,
+        body=body,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+    body: GenerateSyntheticUsersRequest,
+) -> Response[
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+]:
+    """Generate
+
+     Return `num_cases` synthetic-user cases for the authoring UX.
+
+    Args:
+        body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate.
+
+            Generates `num_cases` synthetic-user cases designed to probe
+            `target_specification` against the agent described by `target_task_prompt`,
+            across multi-turn conversations.
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError]
+    """
+
+    kwargs = _get_kwargs(
+        body=body,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: AuthenticatedClient,
+    body: GenerateSyntheticUsersRequest,
+) -> (
+    GenerateSyntheticUsersResponse
+    | GenerateV1SyntheticUserGeneratePostResponse401
+    | GenerateV1SyntheticUserGeneratePostResponse500
+    | GenerateV1SyntheticUserGeneratePostResponse502
+    | HTTPValidationError
+    | None
+):
+    """Generate
+
+     Return `num_cases` synthetic-user cases for the authoring UX.
+
+    Args:
+        body (GenerateSyntheticUsersRequest): Request body for POST /v1/synthetic_user/generate.
+
+            Generates `num_cases` synthetic-user cases designed to probe
+            `target_specification` against the agent described by `target_task_prompt`,
+            across multi-turn conversations.
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        GenerateSyntheticUsersResponse | GenerateV1SyntheticUserGeneratePostResponse401 | GenerateV1SyntheticUserGeneratePostResponse500 | GenerateV1SyntheticUserGeneratePostResponse502 | HTTPValidationError
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+            body=body,
+        )
+    ).parsed

app/desktop/studio_server/api_client/kiln_ai_server_client/models/__init__.py

@@ -55,6 +55,14 @@
 from .generate_batch_input import GenerateBatchInput
 from .generate_batch_output import GenerateBatchOutput
 from .generate_batch_output_data_by_topic import GenerateBatchOutputDataByTopic
+from .generate_synthetic_users_request import GenerateSyntheticUsersRequest
+from .generate_synthetic_users_response import GenerateSyntheticUsersResponse
+from .generate_v1_synthetic_user_generate_post_response_401 import GenerateV1SyntheticUserGeneratePostResponse401
+from .generate_v1_synthetic_user_generate_post_response_500 import GenerateV1SyntheticUserGeneratePostResponse500
+from .generate_v1_synthetic_user_generate_post_response_502 import GenerateV1SyntheticUserGeneratePostResponse502
+from .generate_v1_synthetic_user_generate_post_response_502_code import (
+    GenerateV1SyntheticUserGeneratePostResponse502Code,
+)
 from .get_session_v1_chat_sessions_session_id_get_response_400 import GetSessionV1ChatSessionsSessionIdGetResponse400
 from .get_session_v1_chat_sessions_session_id_get_response_404 import GetSessionV1ChatSessionsSessionIdGetResponse404
 from .get_session_v1_chat_sessions_session_id_get_response_426 import GetSessionV1ChatSessionsSessionIdGetResponse426
@@ -111,6 +119,7 @@
 from .synthetic_data_generation_session_config_input import SyntheticDataGenerationSessionConfigInput
 from .synthetic_data_generation_step_config import SyntheticDataGenerationStepConfig
 from .synthetic_data_generation_step_config_input import SyntheticDataGenerationStepConfigInput
+from .synthetic_user_case import SyntheticUserCase
 from .task_info import TaskInfo
 from .task_metadata import TaskMetadata
 from .task_output import TaskOutput
@@ -167,6 +176,12 @@
     "GenerateBatchInput",
     "GenerateBatchOutput",
     "GenerateBatchOutputDataByTopic",
+    "GenerateSyntheticUsersRequest",
+    "GenerateSyntheticUsersResponse",
+    "GenerateV1SyntheticUserGeneratePostResponse401",
+    "GenerateV1SyntheticUserGeneratePostResponse500",
+    "GenerateV1SyntheticUserGeneratePostResponse502",
+    "GenerateV1SyntheticUserGeneratePostResponse502Code",
     "GetSessionV1ChatSessionsSessionIdGetResponse400",
     "GetSessionV1ChatSessionsSessionIdGetResponse404",
     "GetSessionV1ChatSessionsSessionIdGetResponse426",
@@ -223,6 +238,7 @@
     "SyntheticDataGenerationSessionConfigInput",
     "SyntheticDataGenerationStepConfig",
     "SyntheticDataGenerationStepConfigInput",
+    "SyntheticUserCase",
     "TaskInfo",
     "TaskMetadata",
     "TaskOutput",