Skip to content

Commit 1685f00

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Document Annotations label schema API for AI Observability (DataDog#4012)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 67e85de commit 1685f00

22 files changed

Lines changed: 2354 additions & 27 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 378 additions & 3 deletions
Large diffs are not rendered by default.

‎api/datadog/configuration.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -687,13 +687,15 @@ func NewConfiguration() *Configuration {
687687
"v2.DeleteLLMObsExperiments": false,
688688
"v2.DeleteLLMObsProjects": false,
689689
"v2.GetLLMObsAnnotatedInteractions": false,
690+
"v2.GetLLMObsAnnotationQueueLabelSchema": false,
690691
"v2.GetLLMObsCustomEvalConfig": false,
691692
"v2.ListLLMObsAnnotationQueues": false,
692693
"v2.ListLLMObsDatasetRecords": false,
693694
"v2.ListLLMObsDatasets": false,
694695
"v2.ListLLMObsExperiments": false,
695696
"v2.ListLLMObsProjects": false,
696697
"v2.UpdateLLMObsAnnotationQueue": false,
698+
"v2.UpdateLLMObsAnnotationQueueLabelSchema": false,
697699
"v2.UpdateLLMObsCustomEvalConfig": false,
698700
"v2.UpdateLLMObsDataset": false,
699701
"v2.UpdateLLMObsDatasetRecords": false,

‎api/datadogV2/api_llm_observability.go‎

Lines changed: 203 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@ import (
1919
type LLMObservabilityApi datadog.Service
2020

2121
// CreateLLMObsAnnotationQueue Create an LLM Observability annotation queue.
22-
// Create a new annotation queue. Only `name`, `project_id`, and `description` are accepted.
23-
// Fields such as `created_by`, `owned_by`, `created_at`, `modified_by`, and `modified_at` are inferred by the backend.
22+
// Create an annotation queue. The `name` and `project_id` fields are required.
23+
// An optional `annotation_schema` can be provided to define the labels for the queue.
24+
// Fields such as `created_by`, `owned_by`, `created_at`, `modified_by`,
25+
// and `modified_at` are inferred by the backend.
2426
func (a *LLMObservabilityApi) CreateLLMObsAnnotationQueue(ctx _context.Context, body LLMObsAnnotationQueueRequest) (LLMObsAnnotationQueueResponse, *_nethttp.Response, error) {
2527
var (
2628
localVarHTTPMethod = _nethttp.MethodPost
@@ -1413,6 +1415,102 @@ func (a *LLMObservabilityApi) GetLLMObsAnnotatedInteractions(ctx _context.Contex
14131415
return localVarReturnValue, localVarHTTPResponse, nil
14141416
}
14151417

1418+
// GetLLMObsAnnotationQueueLabelSchema Get annotation queue label schema.
1419+
// Retrieve the label schema for a given annotation queue.
1420+
func (a *LLMObservabilityApi) GetLLMObsAnnotationQueueLabelSchema(ctx _context.Context, queueId string) (LLMObsAnnotationQueueLabelSchemaResponse, *_nethttp.Response, error) {
1421+
var (
1422+
localVarHTTPMethod = _nethttp.MethodGet
1423+
localVarPostBody interface{}
1424+
localVarReturnValue LLMObsAnnotationQueueLabelSchemaResponse
1425+
)
1426+
1427+
operationId := "v2.GetLLMObsAnnotationQueueLabelSchema"
1428+
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
1429+
if !isOperationEnabled {
1430+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
1431+
}
1432+
if isOperationEnabled && a.Client.Cfg.Debug {
1433+
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
1434+
}
1435+
1436+
localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.GetLLMObsAnnotationQueueLabelSchema")
1437+
if err != nil {
1438+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
1439+
}
1440+
1441+
localVarPath := localBasePath + "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/label-schema"
1442+
localVarPath = datadog.ReplacePathParameter(localVarPath, "{queue_id}", _neturl.PathEscape(datadog.ParameterToString(queueId, "")))
1443+
1444+
localVarHeaderParams := make(map[string]string)
1445+
localVarQueryParams := _neturl.Values{}
1446+
localVarFormParams := _neturl.Values{}
1447+
localVarHeaderParams["Accept"] = "application/json"
1448+
1449+
if a.Client.Cfg.DelegatedTokenConfig != nil {
1450+
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
1451+
if err != nil {
1452+
return localVarReturnValue, nil, err
1453+
}
1454+
} else {
1455+
datadog.SetAuthKeys(
1456+
ctx,
1457+
&localVarHeaderParams,
1458+
[2]string{"apiKeyAuth", "DD-API-KEY"},
1459+
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
1460+
)
1461+
}
1462+
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
1463+
if err != nil {
1464+
return localVarReturnValue, nil, err
1465+
}
1466+
1467+
localVarHTTPResponse, err := a.Client.CallAPI(req)
1468+
if err != nil || localVarHTTPResponse == nil {
1469+
return localVarReturnValue, localVarHTTPResponse, err
1470+
}
1471+
1472+
localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
1473+
if err != nil {
1474+
return localVarReturnValue, localVarHTTPResponse, err
1475+
}
1476+
1477+
if localVarHTTPResponse.StatusCode >= 300 {
1478+
newErr := datadog.GenericOpenAPIError{
1479+
ErrorBody: localVarBody,
1480+
ErrorMessage: localVarHTTPResponse.Status,
1481+
}
1482+
if localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 || localVarHTTPResponse.StatusCode == 500 {
1483+
var v JSONAPIErrorResponse
1484+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1485+
if err != nil {
1486+
return localVarReturnValue, localVarHTTPResponse, newErr
1487+
}
1488+
newErr.ErrorModel = v
1489+
return localVarReturnValue, localVarHTTPResponse, newErr
1490+
}
1491+
if localVarHTTPResponse.StatusCode == 429 {
1492+
var v APIErrorResponse
1493+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1494+
if err != nil {
1495+
return localVarReturnValue, localVarHTTPResponse, newErr
1496+
}
1497+
newErr.ErrorModel = v
1498+
}
1499+
return localVarReturnValue, localVarHTTPResponse, newErr
1500+
}
1501+
1502+
err = a.Client.Decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
1503+
if err != nil {
1504+
newErr := datadog.GenericOpenAPIError{
1505+
ErrorBody: localVarBody,
1506+
ErrorMessage: err.Error(),
1507+
}
1508+
return localVarReturnValue, localVarHTTPResponse, newErr
1509+
}
1510+
1511+
return localVarReturnValue, localVarHTTPResponse, nil
1512+
}
1513+
14161514
// GetLLMObsCustomEvalConfig Get a custom evaluator configuration.
14171515
// Retrieve a custom LLM Observability evaluator configuration by its name.
14181516
func (a *LLMObservabilityApi) GetLLMObsCustomEvalConfig(ctx _context.Context, evalName string) (LLMObsCustomEvalConfigResponse, *_nethttp.Response, error) {
@@ -2267,7 +2365,7 @@ func (a *LLMObservabilityApi) ListLLMObsProjects(ctx _context.Context, o ...List
22672365
}
22682366

22692367
// UpdateLLMObsAnnotationQueue Update an LLM Observability annotation queue.
2270-
// Partially update an annotation queue. Only `name` and `description` can be updated.
2368+
// Partially update an annotation queue. The `name`, `description`, and `annotation_schema` fields can be updated.
22712369
func (a *LLMObservabilityApi) UpdateLLMObsAnnotationQueue(ctx _context.Context, queueId string, body LLMObsAnnotationQueueUpdateRequest) (LLMObsAnnotationQueueResponse, *_nethttp.Response, error) {
22722370
var (
22732371
localVarHTTPMethod = _nethttp.MethodPatch
@@ -2365,6 +2463,108 @@ func (a *LLMObservabilityApi) UpdateLLMObsAnnotationQueue(ctx _context.Context,
23652463
return localVarReturnValue, localVarHTTPResponse, nil
23662464
}
23672465

2466+
// UpdateLLMObsAnnotationQueueLabelSchema Update annotation queue label schema.
2467+
// Create or replace the label schema for a given annotation queue.
2468+
// The label schema defines the labels annotators can apply to interactions in the queue.
2469+
// Label names must be unique within the queue and match the pattern `^[a-zA-Z0-9_-]+$`.
2470+
// Each label must have a valid type: score, categorical, boolean, or text.
2471+
func (a *LLMObservabilityApi) UpdateLLMObsAnnotationQueueLabelSchema(ctx _context.Context, queueId string, body LLMObsAnnotationQueueLabelSchemaUpdateRequest) (LLMObsAnnotationQueueLabelSchemaResponse, *_nethttp.Response, error) {
2472+
var (
2473+
localVarHTTPMethod = _nethttp.MethodPut
2474+
localVarPostBody interface{}
2475+
localVarReturnValue LLMObsAnnotationQueueLabelSchemaResponse
2476+
)
2477+
2478+
operationId := "v2.UpdateLLMObsAnnotationQueueLabelSchema"
2479+
isOperationEnabled := a.Client.Cfg.IsUnstableOperationEnabled(operationId)
2480+
if !isOperationEnabled {
2481+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: _fmt.Sprintf("Unstable operation '%s' is disabled", operationId)}
2482+
}
2483+
if isOperationEnabled && a.Client.Cfg.Debug {
2484+
_log.Printf("WARNING: Using unstable operation '%s'", operationId)
2485+
}
2486+
2487+
localBasePath, err := a.Client.Cfg.ServerURLWithContext(ctx, "v2.LLMObservabilityApi.UpdateLLMObsAnnotationQueueLabelSchema")
2488+
if err != nil {
2489+
return localVarReturnValue, nil, datadog.GenericOpenAPIError{ErrorMessage: err.Error()}
2490+
}
2491+
2492+
localVarPath := localBasePath + "/api/v2/llm-obs/v1/annotation-queues/{queue_id}/label-schema"
2493+
localVarPath = datadog.ReplacePathParameter(localVarPath, "{queue_id}", _neturl.PathEscape(datadog.ParameterToString(queueId, "")))
2494+
2495+
localVarHeaderParams := make(map[string]string)
2496+
localVarQueryParams := _neturl.Values{}
2497+
localVarFormParams := _neturl.Values{}
2498+
localVarHeaderParams["Content-Type"] = "application/json"
2499+
localVarHeaderParams["Accept"] = "application/json"
2500+
2501+
// body params
2502+
localVarPostBody = &body
2503+
if a.Client.Cfg.DelegatedTokenConfig != nil {
2504+
err = datadog.UseDelegatedTokenAuth(ctx, &localVarHeaderParams, a.Client.Cfg.DelegatedTokenConfig)
2505+
if err != nil {
2506+
return localVarReturnValue, nil, err
2507+
}
2508+
} else {
2509+
datadog.SetAuthKeys(
2510+
ctx,
2511+
&localVarHeaderParams,
2512+
[2]string{"apiKeyAuth", "DD-API-KEY"},
2513+
[2]string{"appKeyAuth", "DD-APPLICATION-KEY"},
2514+
)
2515+
}
2516+
req, err := a.Client.PrepareRequest(ctx, localVarPath, localVarHTTPMethod, localVarPostBody, localVarHeaderParams, localVarQueryParams, localVarFormParams, nil)
2517+
if err != nil {
2518+
return localVarReturnValue, nil, err
2519+
}
2520+
2521+
localVarHTTPResponse, err := a.Client.CallAPI(req)
2522+
if err != nil || localVarHTTPResponse == nil {
2523+
return localVarReturnValue, localVarHTTPResponse, err
2524+
}
2525+
2526+
localVarBody, err := datadog.ReadBody(localVarHTTPResponse)
2527+
if err != nil {
2528+
return localVarReturnValue, localVarHTTPResponse, err
2529+
}
2530+
2531+
if localVarHTTPResponse.StatusCode >= 300 {
2532+
newErr := datadog.GenericOpenAPIError{
2533+
ErrorBody: localVarBody,
2534+
ErrorMessage: localVarHTTPResponse.Status,
2535+
}
2536+
if localVarHTTPResponse.StatusCode == 400 || localVarHTTPResponse.StatusCode == 401 || localVarHTTPResponse.StatusCode == 403 || localVarHTTPResponse.StatusCode == 404 || localVarHTTPResponse.StatusCode == 500 {
2537+
var v JSONAPIErrorResponse
2538+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
2539+
if err != nil {
2540+
return localVarReturnValue, localVarHTTPResponse, newErr
2541+
}
2542+
newErr.ErrorModel = v
2543+
return localVarReturnValue, localVarHTTPResponse, newErr
2544+
}
2545+
if localVarHTTPResponse.StatusCode == 429 {
2546+
var v APIErrorResponse
2547+
err = a.Client.Decode(&v, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
2548+
if err != nil {
2549+
return localVarReturnValue, localVarHTTPResponse, newErr
2550+
}
2551+
newErr.ErrorModel = v
2552+
}
2553+
return localVarReturnValue, localVarHTTPResponse, newErr
2554+
}
2555+
2556+
err = a.Client.Decode(&localVarReturnValue, localVarBody, localVarHTTPResponse.Header.Get("Content-Type"))
2557+
if err != nil {
2558+
newErr := datadog.GenericOpenAPIError{
2559+
ErrorBody: localVarBody,
2560+
ErrorMessage: err.Error(),
2561+
}
2562+
return localVarReturnValue, localVarHTTPResponse, newErr
2563+
}
2564+
2565+
return localVarReturnValue, localVarHTTPResponse, nil
2566+
}
2567+
23682568
// UpdateLLMObsCustomEvalConfig Create or update a custom evaluator configuration.
23692569
// Create or update a custom LLM Observability evaluator configuration by its name.
23702570
func (a *LLMObservabilityApi) UpdateLLMObsCustomEvalConfig(ctx _context.Context, evalName string, body LLMObsCustomEvalConfigUpdateRequest) (*_nethttp.Response, error) {

‎api/datadogV2/doc.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,13 +461,15 @@
461461
// - [LLMObservabilityApi.DeleteLLMObsExperiments]
462462
// - [LLMObservabilityApi.DeleteLLMObsProjects]
463463
// - [LLMObservabilityApi.GetLLMObsAnnotatedInteractions]
464+
// - [LLMObservabilityApi.GetLLMObsAnnotationQueueLabelSchema]
464465
// - [LLMObservabilityApi.GetLLMObsCustomEvalConfig]
465466
// - [LLMObservabilityApi.ListLLMObsAnnotationQueues]
466467
// - [LLMObservabilityApi.ListLLMObsDatasetRecords]
467468
// - [LLMObservabilityApi.ListLLMObsDatasets]
468469
// - [LLMObservabilityApi.ListLLMObsExperiments]
469470
// - [LLMObservabilityApi.ListLLMObsProjects]
470471
// - [LLMObservabilityApi.UpdateLLMObsAnnotationQueue]
472+
// - [LLMObservabilityApi.UpdateLLMObsAnnotationQueueLabelSchema]
471473
// - [LLMObservabilityApi.UpdateLLMObsCustomEvalConfig]
472474
// - [LLMObservabilityApi.UpdateLLMObsDataset]
473475
// - [LLMObservabilityApi.UpdateLLMObsDatasetRecords]

‎api/datadogV2/model_llm_obs_annotation_queue_data_attributes_request.go‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212

1313
// LLMObsAnnotationQueueDataAttributesRequest Attributes for creating an LLM Observability annotation queue.
1414
type LLMObsAnnotationQueueDataAttributesRequest struct {
15+
// Schema defining the labels for an annotation queue.
16+
AnnotationSchema *LLMObsAnnotationSchema `json:"annotation_schema,omitempty"`
1517
// Description of the annotation queue.
1618
Description *string `json:"description,omitempty"`
1719
// Name of the annotation queue.
@@ -42,6 +44,34 @@ func NewLLMObsAnnotationQueueDataAttributesRequestWithDefaults() *LLMObsAnnotati
4244
return &this
4345
}
4446

47+
// GetAnnotationSchema returns the AnnotationSchema field value if set, zero value otherwise.
48+
func (o *LLMObsAnnotationQueueDataAttributesRequest) GetAnnotationSchema() LLMObsAnnotationSchema {
49+
if o == nil || o.AnnotationSchema == nil {
50+
var ret LLMObsAnnotationSchema
51+
return ret
52+
}
53+
return *o.AnnotationSchema
54+
}
55+
56+
// GetAnnotationSchemaOk returns a tuple with the AnnotationSchema field value if set, nil otherwise
57+
// and a boolean to check if the value has been set.
58+
func (o *LLMObsAnnotationQueueDataAttributesRequest) GetAnnotationSchemaOk() (*LLMObsAnnotationSchema, bool) {
59+
if o == nil || o.AnnotationSchema == nil {
60+
return nil, false
61+
}
62+
return o.AnnotationSchema, true
63+
}
64+
65+
// HasAnnotationSchema returns a boolean if a field has been set.
66+
func (o *LLMObsAnnotationQueueDataAttributesRequest) HasAnnotationSchema() bool {
67+
return o != nil && o.AnnotationSchema != nil
68+
}
69+
70+
// SetAnnotationSchema gets a reference to the given LLMObsAnnotationSchema and assigns it to the AnnotationSchema field.
71+
func (o *LLMObsAnnotationQueueDataAttributesRequest) SetAnnotationSchema(v LLMObsAnnotationSchema) {
72+
o.AnnotationSchema = &v
73+
}
74+
4575
// GetDescription returns the Description field value if set, zero value otherwise.
4676
func (o *LLMObsAnnotationQueueDataAttributesRequest) GetDescription() string {
4777
if o == nil || o.Description == nil {
@@ -122,6 +152,9 @@ func (o LLMObsAnnotationQueueDataAttributesRequest) MarshalJSON() ([]byte, error
122152
if o.UnparsedObject != nil {
123153
return datadog.Marshal(o.UnparsedObject)
124154
}
155+
if o.AnnotationSchema != nil {
156+
toSerialize["annotation_schema"] = o.AnnotationSchema
157+
}
125158
if o.Description != nil {
126159
toSerialize["description"] = o.Description
127160
}
@@ -137,9 +170,10 @@ func (o LLMObsAnnotationQueueDataAttributesRequest) MarshalJSON() ([]byte, error
137170
// UnmarshalJSON deserializes the given payload.
138171
func (o *LLMObsAnnotationQueueDataAttributesRequest) UnmarshalJSON(bytes []byte) (err error) {
139172
all := struct {
140-
Description *string `json:"description,omitempty"`
141-
Name *string `json:"name"`
142-
ProjectId *string `json:"project_id"`
173+
AnnotationSchema *LLMObsAnnotationSchema `json:"annotation_schema,omitempty"`
174+
Description *string `json:"description,omitempty"`
175+
Name *string `json:"name"`
176+
ProjectId *string `json:"project_id"`
143177
}{}
144178
if err = datadog.Unmarshal(bytes, &all); err != nil {
145179
return datadog.Unmarshal(bytes, &o.UnparsedObject)
@@ -152,10 +186,16 @@ func (o *LLMObsAnnotationQueueDataAttributesRequest) UnmarshalJSON(bytes []byte)
152186
}
153187
additionalProperties := make(map[string]interface{})
154188
if err = datadog.Unmarshal(bytes, &additionalProperties); err == nil {
155-
datadog.DeleteKeys(additionalProperties, &[]string{"description", "name", "project_id"})
189+
datadog.DeleteKeys(additionalProperties, &[]string{"annotation_schema", "description", "name", "project_id"})
156190
} else {
157191
return err
158192
}
193+
194+
hasInvalidField := false
195+
if all.AnnotationSchema != nil && all.AnnotationSchema.UnparsedObject != nil && o.UnparsedObject == nil {
196+
hasInvalidField = true
197+
}
198+
o.AnnotationSchema = all.AnnotationSchema
159199
o.Description = all.Description
160200
o.Name = *all.Name
161201
o.ProjectId = *all.ProjectId
@@ -164,5 +204,9 @@ func (o *LLMObsAnnotationQueueDataAttributesRequest) UnmarshalJSON(bytes []byte)
164204
o.AdditionalProperties = additionalProperties
165205
}
166206

207+
if hasInvalidField {
208+
return datadog.Unmarshal(bytes, &o.UnparsedObject)
209+
}
210+
167211
return nil
168212
}

0 commit comments

Comments
 (0)