Skip to content

Commit a34e4f8

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for LLM Observability eval config CRUD endpoints (#3980)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 0531b50 commit a34e4f8

33 files changed

Lines changed: 5439 additions & 25 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 651 additions & 0 deletions
Large diffs are not rendered by default.

‎api/datadog/configuration.go‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,33 @@ func NewConfiguration() *Configuration {
667667
"v2.ListFleetTracers": false,
668668
"v2.TriggerFleetSchedule": false,
669669
"v2.UpdateFleetSchedule": false,
670+
"v2.CreateLLMObsAnnotationQueue": false,
671+
"v2.CreateLLMObsAnnotationQueueInteractions": false,
672+
"v2.CreateLLMObsDataset": false,
673+
"v2.CreateLLMObsDatasetRecords": false,
674+
"v2.CreateLLMObsExperiment": false,
675+
"v2.CreateLLMObsExperimentEvents": false,
676+
"v2.CreateLLMObsProject": false,
677+
"v2.DeleteLLMObsAnnotationQueue": false,
678+
"v2.DeleteLLMObsAnnotationQueueInteractions": false,
679+
"v2.DeleteLLMObsCustomEvalConfig": false,
680+
"v2.DeleteLLMObsDatasetRecords": false,
681+
"v2.DeleteLLMObsDatasets": false,
682+
"v2.DeleteLLMObsExperiments": false,
683+
"v2.DeleteLLMObsProjects": false,
684+
"v2.GetLLMObsAnnotatedInteractions": false,
685+
"v2.GetLLMObsCustomEvalConfig": false,
686+
"v2.ListLLMObsAnnotationQueues": false,
687+
"v2.ListLLMObsDatasetRecords": false,
688+
"v2.ListLLMObsDatasets": false,
689+
"v2.ListLLMObsExperiments": false,
690+
"v2.ListLLMObsProjects": false,
691+
"v2.UpdateLLMObsAnnotationQueue": false,
692+
"v2.UpdateLLMObsCustomEvalConfig": false,
693+
"v2.UpdateLLMObsDataset": false,
694+
"v2.UpdateLLMObsDatasetRecords": false,
695+
"v2.UpdateLLMObsExperiment": false,
696+
"v2.UpdateLLMObsProject": false,
670697
"v2.CreateOpenAPI": false,
671698
"v2.DeleteOpenAPI": false,
672699
"v2.GetOpenAPI": false,
@@ -820,30 +847,6 @@ func NewConfiguration() *Configuration {
820847
"v2.UpdateJiraIssueTemplate": false,
821848
"v2.CreateTenancyConfig": false,
822849
"v2.GetTenancyConfigs": false,
823-
"v2.CreateLLMObsAnnotationQueue": false,
824-
"v2.CreateLLMObsAnnotationQueueInteractions": false,
825-
"v2.CreateLLMObsDataset": false,
826-
"v2.CreateLLMObsDatasetRecords": false,
827-
"v2.CreateLLMObsExperiment": false,
828-
"v2.CreateLLMObsExperimentEvents": false,
829-
"v2.CreateLLMObsProject": false,
830-
"v2.DeleteLLMObsAnnotationQueue": false,
831-
"v2.DeleteLLMObsAnnotationQueueInteractions": false,
832-
"v2.DeleteLLMObsDatasetRecords": false,
833-
"v2.DeleteLLMObsDatasets": false,
834-
"v2.DeleteLLMObsExperiments": false,
835-
"v2.DeleteLLMObsProjects": false,
836-
"v2.GetLLMObsAnnotatedInteractions": false,
837-
"v2.ListLLMObsAnnotationQueues": false,
838-
"v2.ListLLMObsDatasetRecords": false,
839-
"v2.ListLLMObsDatasets": false,
840-
"v2.ListLLMObsExperiments": false,
841-
"v2.ListLLMObsProjects": false,
842-
"v2.UpdateLLMObsAnnotationQueue": false,
843-
"v2.UpdateLLMObsDataset": false,
844-
"v2.UpdateLLMObsDatasetRecords": false,
845-
"v2.UpdateLLMObsExperiment": false,
846-
"v2.UpdateLLMObsProject": false,
847850
"v2.AddRoleToRestrictionQuery": false,
848851
"v2.CreateRestrictionQuery": false,
849852
"v2.DeleteRestrictionQuery": false,

‎api/datadogV2/api_llm_observability.go‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,92 @@ func (a *LLMObservabilityApi) DeleteLLMObsAnnotationQueueInteractions(ctx _conte
876876
return localVarHTTPResponse, nil
877877
}
878878

879+
// DeleteLLMObsCustomEvalConfig Delete a custom evaluator configuration.
880+
// Delete a custom LLM Observability evaluator configuration by its name.
881+
func (a *LLMObservabilityApi) DeleteLLMObsCustomEvalConfig(ctx _context.Context, evalName string) (*_nethttp.Response, error) {
882+
var (
883+
localVarHTTPMethod = _nethttp.MethodDelete
884+
localVarPostBody interface{}
885+
)
886+
887+
operationId := "v2.DeleteLLMObsCustomEvalConfig"
888+
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
889+
if !isOperationEnabled {
890+
return nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
891+
}
892+
if isOperationEnabled && a.Client.Cfg.Debug {
893+
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
894+
}
895+
896+
localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.DeleteLLMObsCustomEvalConfig")
897+
if err != nil {
898+
return nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
899+
}
900+
901+
localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
902+
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))
903+
904+
localVarHeaderParams := make(map[string]string)
905+
localVarQueryParams := _neturl.Values{}
906+
localVarFormParams := _neturl.Values{}
907+
localVarHeaderParams["Accept"] = "*/*"
908+
909+
if a.Client.Cfg.DelegatedTokenConfig != nil {
910+
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
911+
if err != nil {
912+
return nil, err
913+
}
914+
} else {
915+
datadog.SetAuthKeys(
916+
ctx,
917+
&localVarHeaderParams,
918+
[2]string{"apiKeyAuth", "DD-API-KEY"},
919+
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
920+
)
921+
}
922+
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
923+
if err != nil {
924+
return nil, err
925+
}
926+
927+
localVarHTTPResponse, err := a.Client.CallAPI(req)
928+
if err != nil || localVarHTTPResponse == nil {
929+
return localVarHTTPResponse, err
930+
}
931+
932+
localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
933+
if err != nil {
934+
return localVarHTTPResponse, err
935+
}
936+
937+
if localVarHTTPResponse.StatusCode >= 300 {
938+
newErr := datadog.GenericOpenAPIError{
939+
ErrorBody: localVarBody,
940+
ErrorMessage: localVarHTTPResponse.Status,
941+
}
942+
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 {
943+
var v JSONAPIErrorResponse
944+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
945+
if err != nil {
946+
return localVarHTTPResponse, newErr
947+
}
948+
newErr.ErrorModel = v
949+
return localVarHTTPResponse, newErr
950+
}
951+
if localVarHTTPResponse.StatusCode == 429 {
952+
var v APIErrorResponse
953+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
954+
if err != nil {
955+
return localVarHTTPResponse, newErr
956+
}
957+
newErr.ErrorModel = v
958+
}
959+
return localVarHTTPResponse, newErr
960+
}
961+
962+
return localVarHTTPResponse, nil
963+
}
964+
879965
// DeleteLLMObsDatasetRecords Delete LLM Observability dataset records.
880966
// Delete one or more records from an LLM Observability dataset.
881967
func (a *LLMObservabilityApi) DeleteLLMObsDatasetRecords(ctx _context.Context, projectId string, datasetId string, body LLMObsDeleteDatasetRecordsRequest) (*_nethttp.Response, error) {
@@ -1327,6 +1413,102 @@ func (a *LLMObservabilityApi) GetLLMObsAnnotatedInteractions(ctx _context.Contex
13271413
return localVarReturnValue, localVarHTTPResponse, nil
13281414
}
13291415

1416+
// GetLLMObsCustomEvalConfig Get a custom evaluator configuration.
1417+
// Retrieve a custom LLM Observability evaluator configuration by its name.
1418+
func (a *LLMObservabilityApi) GetLLMObsCustomEvalConfig(ctx _context.Context, evalName string) (LLMObsCustomEvalConfigResponse, *_nethttp.Response, error) {
1419+
var (
1420+
localVarHTTPMethod = _nethttp.MethodGet
1421+
localVarPostBody interface{}
1422+
localVarReturnValue LLMObsCustomEvalConfigResponse
1423+
)
1424+
1425+
operationId := "v2.GetLLMObsCustomEvalConfig"
1426+
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
1427+
if !isOperationEnabled {
1428+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
1429+
}
1430+
if isOperationEnabled && a.Client.Cfg.Debug {
1431+
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
1432+
}
1433+
1434+
localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.GetLLMObsCustomEvalConfig")
1435+
if err != nil {
1436+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
1437+
}
1438+
1439+
localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
1440+
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))
1441+
1442+
localVarHeaderParams := make(map[string]string)
1443+
localVarQueryParams := _neturl.Values{}
1444+
localVarFormParams := _neturl.Values{}
1445+
localVarHeaderParams["Accept"] = "application/json"
1446+
1447+
if a.Client.Cfg.DelegatedTokenConfig != nil {
1448+
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
1449+
if err != nil {
1450+
return localVarReturnValue, nil, err
1451+
}
1452+
} else {
1453+
datadog.SetAuthKeys(
1454+
ctx,
1455+
&localVarHeaderParams,
1456+
[2]string{"apiKeyAuth", "DD-API-KEY"},
1457+
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
1458+
)
1459+
}
1460+
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
1461+
if err != nil {
1462+
return localVarReturnValue, nil, err
1463+
}
1464+
1465+
localVarHTTPResponse, err := a.Client.CallAPI(req)
1466+
if err != nil || localVarHTTPResponse == nil {
1467+
return localVarReturnValue, localVarHTTPResponse, err
1468+
}
1469+
1470+
localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
1471+
if err != nil {
1472+
return localVarReturnValue, localVarHTTPResponse, err
1473+
}
1474+
1475+
if localVarHTTPResponse.StatusCode >= 300 {
1476+
newErr := datadog.GenericOpenAPIError{
1477+
ErrorBody: localVarBody,
1478+
ErrorMessage: localVarHTTPResponse.Status,
1479+
}
1480+
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 {
1481+
var v JSONAPIErrorResponse
1482+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1483+
if err != nil {
1484+
return localVarReturnValue, localVarHTTPResponse, newErr
1485+
}
1486+
newErr.ErrorModel = v
1487+
return localVarReturnValue, localVarHTTPResponse, newErr
1488+
}
1489+
if localVarHTTPResponse.StatusCode == 429 {
1490+
var v APIErrorResponse
1491+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1492+
if err != nil {
1493+
return localVarReturnValue, localVarHTTPResponse, newErr
1494+
}
1495+
newErr.ErrorModel = v
1496+
}
1497+
return localVarReturnValue, localVarHTTPResponse, newErr
1498+
}
1499+
1500+
err = a.Client.Decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1501+
if err != nil {
1502+
newErr := datadog.GenericOpenAPIError{
1503+
ErrorBody: localVarBody,
1504+
ErrorMessage: err.Error(),
1505+
}
1506+
return localVarReturnValue, localVarHTTPResponse, newErr
1507+
}
1508+
1509+
return localVarReturnValue, localVarHTTPResponse, nil
1510+
}
1511+
13301512
// ListLLMObsAnnotationQueuesOptionalParameters holds optional parameters for ListLLMObsAnnotationQueues.
13311513
type ListLLMObsAnnotationQueuesOptionalParameters struct {
13321514
ProjectId *string
@@ -2183,6 +2365,95 @@ func (a *LLMObservabilityApi) UpdateLLMObsAnnotationQueue(ctx _context.Context,
21832365
return localVarReturnValue, localVarHTTPResponse, nil
21842366
}
21852367

2368+
// UpdateLLMObsCustomEvalConfig Create or update a custom evaluator configuration.
2369+
// Create or update a custom LLM Observability evaluator configuration by its name.
2370+
func (a *LLMObservabilityApi) UpdateLLMObsCustomEvalConfig(ctx _context.Context, evalName string, body LLMObsCustomEvalConfigUpdateRequest) (*_nethttp.Response, error) {
2371+
var (
2372+
localVarHTTPMethod = _nethttp.MethodPut
2373+
localVarPostBody interface{}
2374+
)
2375+
2376+
operationId := "v2.UpdateLLMObsCustomEvalConfig"
2377+
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
2378+
if !isOperationEnabled {
2379+
return nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
2380+
}
2381+
if isOperationEnabled && a.Client.Cfg.Debug {
2382+
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
2383+
}
2384+
2385+
localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.UpdateLLMObsCustomEvalConfig")
2386+
if err != nil {
2387+
return nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
2388+
}
2389+
2390+
localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
2391+
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))
2392+
2393+
localVarHeaderParams := make(map[string]string)
2394+
localVarQueryParams := _neturl.Values{}
2395+
localVarFormParams := _neturl.Values{}
2396+
localVarHeaderParams["Content-Type"] = "application/json"
2397+
localVarHeaderParams["Accept"] = "*/*"
2398+
2399+
// body params
2400+
localVarPostBody = &body
2401+
if a.Client.Cfg.DelegatedTokenConfig != nil {
2402+
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
2403+
if err != nil {
2404+
return nil, err
2405+
}
2406+
} else {
2407+
datadog.SetAuthKeys(
2408+
ctx,
2409+
&localVarHeaderParams,
2410+
[2]string{"apiKeyAuth", "DD-API-KEY"},
2411+
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
2412+
)
2413+
}
2414+
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
2415+
if err != nil {
2416+
return nil, err
2417+
}
2418+
2419+
localVarHTTPResponse, err := a.Client.CallAPI(req)
2420+
if err != nil || localVarHTTPResponse == nil {
2421+
return localVarHTTPResponse, err
2422+
}
2423+
2424+
localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
2425+
if err != nil {
2426+
return localVarHTTPResponse, err
2427+
}
2428+
2429+
if localVarHTTPResponse.StatusCode >= 300 {
2430+
newErr := datadog.GenericOpenAPIError{
2431+
ErrorBody: localVarBody,
2432+
ErrorMessage: localVarHTTPResponse.Status,
2433+
}
2434+
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 || localVarHTTPResponse.StatusCode == 422 {
2435+
var v JSONAPIErrorResponse
2436+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
2437+
if err != nil {
2438+
return localVarHTTPResponse, newErr
2439+
}
2440+
newErr.ErrorModel = v
2441+
return localVarHTTPResponse, newErr
2442+
}
2443+
if localVarHTTPResponse.StatusCode == 429 {
2444+
var v APIErrorResponse
2445+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
2446+
if err != nil {
2447+
return localVarHTTPResponse, newErr
2448+
}
2449+
newErr.ErrorModel = v
2450+
}
2451+
return localVarHTTPResponse, newErr
2452+
}
2453+
2454+
return localVarHTTPResponse, nil
2455+
}
2456+
21862457
// UpdateLLMObsDataset Update an LLM Observability dataset.
21872458
// Partially update an existing LLM Observability dataset within the specified project.
21882459
func (a *LLMObservabilityApi) UpdateLLMObsDataset(ctx _context.Context, projectId string, datasetId string, body LLMObsDatasetUpdateRequest) (LLMObsDatasetResponse, *_nethttp.Response, error) {

‎api/datadogV2/doc.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,17 +455,20 @@
455455
// - [LLMObservabilityApi.CreateLLMObsProject]
456456
// - [LLMObservabilityApi.DeleteLLMObsAnnotationQueue]
457457
// - [LLMObservabilityApi.DeleteLLMObsAnnotationQueueInteractions]
458+
// - [LLMObservabilityApi.DeleteLLMObsCustomEvalConfig]
458459
// - [LLMObservabilityApi.DeleteLLMObsDatasetRecords]
459460
// - [LLMObservabilityApi.DeleteLLMObsDatasets]
460461
// - [LLMObservabilityApi.DeleteLLMObsExperiments]
461462
// - [LLMObservabilityApi.DeleteLLMObsProjects]
462463
// - [LLMObservabilityApi.GetLLMObsAnnotatedInteractions]
464+
// - [LLMObservabilityApi.GetLLMObsCustomEvalConfig]
463465
// - [LLMObservabilityApi.ListLLMObsAnnotationQueues]
464466
// - [LLMObservabilityApi.ListLLMObsDatasetRecords]
465467
// - [LLMObservabilityApi.ListLLMObsDatasets]
466468
// - [LLMObservabilityApi.ListLLMObsExperiments]
467469
// - [LLMObservabilityApi.ListLLMObsProjects]
468470
// - [LLMObservabilityApi.UpdateLLMObsAnnotationQueue]
471+
// - [LLMObservabilityApi.UpdateLLMObsCustomEvalConfig]
469472
// - [LLMObservabilityApi.UpdateLLMObsDataset]
470473
// - [LLMObservabilityApi.UpdateLLMObsDatasetRecords]
471474
// - [LLMObservabilityApi.UpdateLLMObsExperiment]

0 commit comments

Comments
 (0)