Skip to content

Add OpenAPI documentation for LLM Observability eval config CRUD endpoints #3980

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
api-clients-generation-pipeline merged 1 commit into from
Apr 20, 2026
Merged

Add OpenAPI documentation for LLM Observability eval config CRUD endpoints #3980

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
651 changes: 651 additions & 0 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop

Large diffs are not rendered by default.

51 changes: 27 additions & 24 deletions api/datadog/configuration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,33 @@ func NewConfiguration() *Configuration {
"v2.ListFleetTracers": false,
"v2.TriggerFleetSchedule": false,
"v2.UpdateFleetSchedule": false,
"v2.CreateLLMObsAnnotationQueue": false,
"v2.CreateLLMObsAnnotationQueueInteractions": false,
"v2.CreateLLMObsDataset": false,
"v2.CreateLLMObsDatasetRecords": false,
"v2.CreateLLMObsExperiment": false,
"v2.CreateLLMObsExperimentEvents": false,
"v2.CreateLLMObsProject": false,
"v2.DeleteLLMObsAnnotationQueue": false,
"v2.DeleteLLMObsAnnotationQueueInteractions": false,
"v2.DeleteLLMObsCustomEvalConfig": false,
"v2.DeleteLLMObsDatasetRecords": false,
"v2.DeleteLLMObsDatasets": false,
"v2.DeleteLLMObsExperiments": false,
"v2.DeleteLLMObsProjects": false,
"v2.GetLLMObsAnnotatedInteractions": false,
"v2.GetLLMObsCustomEvalConfig": false,
"v2.ListLLMObsAnnotationQueues": false,
"v2.ListLLMObsDatasetRecords": false,
"v2.ListLLMObsDatasets": false,
"v2.ListLLMObsExperiments": false,
"v2.ListLLMObsProjects": false,
"v2.UpdateLLMObsAnnotationQueue": false,
"v2.UpdateLLMObsCustomEvalConfig": false,
"v2.UpdateLLMObsDataset": false,
"v2.UpdateLLMObsDatasetRecords": false,
"v2.UpdateLLMObsExperiment": false,
"v2.UpdateLLMObsProject": false,
"v2.CreateOpenAPI": false,
"v2.DeleteOpenAPI": false,
"v2.GetOpenAPI": false,
Expand Down Expand Up @@ -820,30 +847,6 @@ func NewConfiguration() *Configuration {
"v2.UpdateJiraIssueTemplate": false,
"v2.CreateTenancyConfig": false,
"v2.GetTenancyConfigs": false,
"v2.CreateLLMObsAnnotationQueue": false,
"v2.CreateLLMObsAnnotationQueueInteractions": false,
"v2.CreateLLMObsDataset": false,
"v2.CreateLLMObsDatasetRecords": false,
"v2.CreateLLMObsExperiment": false,
"v2.CreateLLMObsExperimentEvents": false,
"v2.CreateLLMObsProject": false,
"v2.DeleteLLMObsAnnotationQueue": false,
"v2.DeleteLLMObsAnnotationQueueInteractions": false,
"v2.DeleteLLMObsDatasetRecords": false,
"v2.DeleteLLMObsDatasets": false,
"v2.DeleteLLMObsExperiments": false,
"v2.DeleteLLMObsProjects": false,
"v2.GetLLMObsAnnotatedInteractions": false,
"v2.ListLLMObsAnnotationQueues": false,
"v2.ListLLMObsDatasetRecords": false,
"v2.ListLLMObsDatasets": false,
"v2.ListLLMObsExperiments": false,
"v2.ListLLMObsProjects": false,
"v2.UpdateLLMObsAnnotationQueue": false,
"v2.UpdateLLMObsDataset": false,
"v2.UpdateLLMObsDatasetRecords": false,
"v2.UpdateLLMObsExperiment": false,
"v2.UpdateLLMObsProject": false,
"v2.AddRoleToRestrictionQuery": false,
"v2.CreateRestrictionQuery": false,
"v2.DeleteRestrictionQuery": false,
Expand Down
271 changes: 271 additions & 0 deletions api/datadogV2/api_llm_observability.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,92 @@ func (a *LLMObservabilityApi) DeleteLLMObsAnnotationQueueInteractions(ctx _conte
return localVarHTTPResponse, nil
}

// DeleteLLMObsCustomEvalConfig Delete a custom evaluator configuration.
// Delete a custom LLM Observability evaluator configuration by its name.
func (a *LLMObservabilityApi) DeleteLLMObsCustomEvalConfig(ctx _context.Context, evalName string) (*_nethttp.Response, error) {
var (
localVarHTTPMethod = _nethttp.MethodDelete
localVarPostBody interface{}
)

operationId := "v2.DeleteLLMObsCustomEvalConfig"
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
if !isOperationEnabled {
return nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
}
if isOperationEnabled && a.Client.Cfg.Debug {
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
}

localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.DeleteLLMObsCustomEvalConfig")
if err != nil {
return nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
}

localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))

localVarHeaderParams := make(map[string]string)
localVarQueryParams := _neturl.Values{}
localVarFormParams := _neturl.Values{}
localVarHeaderParams["Accept"] = "*/*"

if a.Client.Cfg.DelegatedTokenConfig != nil {
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
if err != nil {
return nil, err
}
} else {
datadog.SetAuthKeys(
ctx,
&localVarHeaderParams,
[2]string{"apiKeyAuth", "DD-API-KEY"},
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
)
}
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
if err != nil {
return nil, err
}

localVarHTTPResponse, err := a.Client.CallAPI(req)
if err != nil || localVarHTTPResponse == nil {
return localVarHTTPResponse, err
}

localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
if err != nil {
return localVarHTTPResponse, err
}

if localVarHTTPResponse.StatusCode >= 300 {
newErr := datadog.GenericOpenAPIError{
ErrorBody: localVarBody,
ErrorMessage: localVarHTTPResponse.Status,
}
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 {
var v JSONAPIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
return localVarHTTPResponse, newErr
}
if localVarHTTPResponse.StatusCode == 429 {
var v APIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
}
return localVarHTTPResponse, newErr
}

return localVarHTTPResponse, nil
}

// DeleteLLMObsDatasetRecords Delete LLM Observability dataset records.
// Delete one or more records from an LLM Observability dataset.
func (a *LLMObservabilityApi) DeleteLLMObsDatasetRecords(ctx _context.Context, projectId string, datasetId string, body LLMObsDeleteDatasetRecordsRequest) (*_nethttp.Response, error) {
Expand Down Expand Up @@ -1327,6 +1413,102 @@ func (a *LLMObservabilityApi) GetLLMObsAnnotatedInteractions(ctx _context.Contex
return localVarReturnValue, localVarHTTPResponse, nil
}

// GetLLMObsCustomEvalConfig Get a custom evaluator configuration.
// Retrieve a custom LLM Observability evaluator configuration by its name.
func (a *LLMObservabilityApi) GetLLMObsCustomEvalConfig(ctx _context.Context, evalName string) (LLMObsCustomEvalConfigResponse, *_nethttp.Response, error) {
var (
localVarHTTPMethod = _nethttp.MethodGet
localVarPostBody interface{}
localVarReturnValue LLMObsCustomEvalConfigResponse
)

operationId := "v2.GetLLMObsCustomEvalConfig"
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
if !isOperationEnabled {
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
}
if isOperationEnabled && a.Client.Cfg.Debug {
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
}

localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.GetLLMObsCustomEvalConfig")
if err != nil {
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
}

localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))

localVarHeaderParams := make(map[string]string)
localVarQueryParams := _neturl.Values{}
localVarFormParams := _neturl.Values{}
localVarHeaderParams["Accept"] = "application/json"

if a.Client.Cfg.DelegatedTokenConfig != nil {
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
if err != nil {
return localVarReturnValue, nil, err
}
} else {
datadog.SetAuthKeys(
ctx,
&localVarHeaderParams,
[2]string{"apiKeyAuth", "DD-API-KEY"},
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
)
}
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
if err != nil {
return localVarReturnValue, nil, err
}

localVarHTTPResponse, err := a.Client.CallAPI(req)
if err != nil || localVarHTTPResponse == nil {
return localVarReturnValue, localVarHTTPResponse, err
}

localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
if err != nil {
return localVarReturnValue, localVarHTTPResponse, err
}

if localVarHTTPResponse.StatusCode >= 300 {
newErr := datadog.GenericOpenAPIError{
ErrorBody: localVarBody,
ErrorMessage: localVarHTTPResponse.Status,
}
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 {
var v JSONAPIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarReturnValue, localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
return localVarReturnValue, localVarHTTPResponse, newErr
}
if localVarHTTPResponse.StatusCode == 429 {
var v APIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarReturnValue, localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
}
return localVarReturnValue, localVarHTTPResponse, newErr
}

err = a.Client.Decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
newErr := datadog.GenericOpenAPIError{
ErrorBody: localVarBody,
ErrorMessage: err.Error(),
}
return localVarReturnValue, localVarHTTPResponse, newErr
}

return localVarReturnValue, localVarHTTPResponse, nil
}

// ListLLMObsAnnotationQueuesOptionalParameters holds optional parameters for ListLLMObsAnnotationQueues.
type ListLLMObsAnnotationQueuesOptionalParameters struct {
ProjectId *string
Expand Down Expand Up @@ -2183,6 +2365,95 @@ func (a *LLMObservabilityApi) UpdateLLMObsAnnotationQueue(ctx _context.Context,
return localVarReturnValue, localVarHTTPResponse, nil
}

// UpdateLLMObsCustomEvalConfig Create or update a custom evaluator configuration.
// Create or update a custom LLM Observability evaluator configuration by its name.
func (a *LLMObservabilityApi) UpdateLLMObsCustomEvalConfig(ctx _context.Context, evalName string, body LLMObsCustomEvalConfigUpdateRequest) (*_nethttp.Response, error) {
var (
localVarHTTPMethod = _nethttp.MethodPut
localVarPostBody interface{}
)

operationId := "v2.UpdateLLMObsCustomEvalConfig"
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
if !isOperationEnabled {
return nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
}
if isOperationEnabled && a.Client.Cfg.Debug {
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
}

localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.UpdateLLMObsCustomEvalConfig")
if err != nil {
return nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
}

localVarPath := localBasePath + "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}"
localVarPath = datadog.ReplacePathParameter(localVarPath, "{eval_name}", _neturl.PathEscape(datadog.ParameterToString(evalName, "")))

localVarHeaderParams := make(map[string]string)
localVarQueryParams := _neturl.Values{}
localVarFormParams := _neturl.Values{}
localVarHeaderParams["Content-Type"] = "application/json"
localVarHeaderParams["Accept"] = "*/*"

// body params
localVarPostBody = &body
if a.Client.Cfg.DelegatedTokenConfig != nil {
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
if err != nil {
return nil, err
}
} else {
datadog.SetAuthKeys(
ctx,
&localVarHeaderParams,
[2]string{"apiKeyAuth", "DD-API-KEY"},
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
)
}
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
if err != nil {
return nil, err
}

localVarHTTPResponse, err := a.Client.CallAPI(req)
if err != nil || localVarHTTPResponse == nil {
return localVarHTTPResponse, err
}

localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
if err != nil {
return localVarHTTPResponse, err
}

if localVarHTTPResponse.StatusCode >= 300 {
newErr := datadog.GenericOpenAPIError{
ErrorBody: localVarBody,
ErrorMessage: localVarHTTPResponse.Status,
}
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 || localVarHTTPResponse.StatusCode == 422 {
var v JSONAPIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
return localVarHTTPResponse, newErr
}
if localVarHTTPResponse.StatusCode == 429 {
var v APIErrorResponse
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
if err != nil {
return localVarHTTPResponse, newErr
}
newErr.ErrorModel = v
}
return localVarHTTPResponse, newErr
}

return localVarHTTPResponse, nil
}

// UpdateLLMObsDataset Update an LLM Observability dataset.
// Partially update an existing LLM Observability dataset within the specified project.
func (a *LLMObservabilityApi) UpdateLLMObsDataset(ctx _context.Context, projectId string, datasetId string, body LLMObsDatasetUpdateRequest) (LLMObsDatasetResponse, *_nethttp.Response, error) {
Expand Down
3 changes: 3 additions & 0 deletions api/datadogV2/doc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,17 +455,20 @@
// - [LLMObservabilityApi.CreateLLMObsProject]
// - [LLMObservabilityApi.DeleteLLMObsAnnotationQueue]
// - [LLMObservabilityApi.DeleteLLMObsAnnotationQueueInteractions]
// - [LLMObservabilityApi.DeleteLLMObsCustomEvalConfig]
// - [LLMObservabilityApi.DeleteLLMObsDatasetRecords]
// - [LLMObservabilityApi.DeleteLLMObsDatasets]
// - [LLMObservabilityApi.DeleteLLMObsExperiments]
// - [LLMObservabilityApi.DeleteLLMObsProjects]
// - [LLMObservabilityApi.GetLLMObsAnnotatedInteractions]
// - [LLMObservabilityApi.GetLLMObsCustomEvalConfig]
// - [LLMObservabilityApi.ListLLMObsAnnotationQueues]
// - [LLMObservabilityApi.ListLLMObsDatasetRecords]
// - [LLMObservabilityApi.ListLLMObsDatasets]
// - [LLMObservabilityApi.ListLLMObsExperiments]
// - [LLMObservabilityApi.ListLLMObsProjects]
// - [LLMObservabilityApi.UpdateLLMObsAnnotationQueue]
// - [LLMObservabilityApi.UpdateLLMObsCustomEvalConfig]
// - [LLMObservabilityApi.UpdateLLMObsDataset]
// - [LLMObservabilityApi.UpdateLLMObsDatasetRecords]
// - [LLMObservabilityApi.UpdateLLMObsExperiment]
Expand Down
Loading
Loading