Skip to content

Commit 7e7a9f9

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add display_block interaction type to LLM Obs annotation queues (#4142)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 04d22df commit 7e7a9f9

21 files changed

Lines changed: 1153 additions & 271 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 350 additions & 68 deletions
Large diffs are not rendered by default.

‎features/v2/llm_observability.feature‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,24 @@ Feature: LLM Observability
88
And a valid "appKeyAuth" key in the system
99
And an instance of "LLMObservability" API
1010

11+
@skip @team:DataDog/ml-observability
12+
Scenario: Add a display_block interaction returns "Created" response
13+
Given operation "CreateLLMObsAnnotationQueueInteractions" enabled
14+
And new "CreateLLMObsAnnotationQueueInteractions" request
15+
And request contains "queue_id" parameter from "REPLACE.ME"
16+
And body with value {"data": {"attributes": {"interactions": [{"type": "display_block", "display_block": [{"type": "markdown", "content": "## Triage Instructions"}]}]}, "type": "interactions"}}
17+
When the request is sent
18+
Then the response status is 201 Created
19+
20+
@skip @team:DataDog/ml-observability
21+
Scenario: Add a display_block interaction with an image block missing url returns "Bad Request" response
22+
Given operation "CreateLLMObsAnnotationQueueInteractions" enabled
23+
And new "CreateLLMObsAnnotationQueueInteractions" request
24+
And request contains "queue_id" parameter from "REPLACE.ME"
25+
And body with value {"data": {"attributes": {"interactions": [{"type": "display_block", "display_block": [{"type": "image"}]}]}, "type": "interactions"}}
26+
When the request is sent
27+
Then the response status is 400 Bad Request
28+
1129
@generated @skip @team:DataDog/ml-observability
1230
Scenario: Add annotation queue interactions returns "Bad Request" response
1331
Given operation "CreateLLMObsAnnotationQueueInteractions" enabled

‎services/llm_observability/src/v2/LLMObservabilityApi.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4775,8 +4775,16 @@ export class LLMObservabilityApi {
47754775
}
47764776

47774777
/**
4778-
* Add one or more interactions (traces or sessions) to an annotation queue.
4779-
* At least one interaction must be provided.
4778+
* Add one or more interactions to an annotation queue. At least one
4779+
* interaction must be provided. Each interaction has a `type`:
4780+
*
4781+
* - `trace`, `experiment_trace`, `session`: `content_id` references the
4782+
* upstream entity; the server fetches the actual content.
4783+
* - `display_block`: omit `content_id` and provide the rendered content
4784+
* in `display_block`. The server generates `content_id` as a
4785+
* deterministic hash of the block list.
4786+
*
4787+
* Items of different types can be mixed in a single request.
47804788
* @param param The request object
47814789
*/
47824790
public createLLMObsAnnotationQueueInteractions(

‎services/llm_observability/src/v2/index.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ export { LLMObsAnnotationQueueUpdateDataAttributesRequest } from "./models/LLMOb
7070
export { LLMObsAnnotationQueueUpdateDataRequest } from "./models/LLMObsAnnotationQueueUpdateDataRequest";
7171
export { LLMObsAnnotationQueueUpdateRequest } from "./models/LLMObsAnnotationQueueUpdateRequest";
7272
export { LLMObsAnnotationSchema } from "./models/LLMObsAnnotationSchema";
73+
export { LLMObsContentBlock } from "./models/LLMObsContentBlock";
74+
export { LLMObsContentBlockHeaderLevel } from "./models/LLMObsContentBlockHeaderLevel";
75+
export { LLMObsContentBlockLLMObsTraceInteractionType } from "./models/LLMObsContentBlockLLMObsTraceInteractionType";
76+
export { LLMObsContentBlockTimeFrame } from "./models/LLMObsContentBlockTimeFrame";
77+
export { LLMObsContentBlockType } from "./models/LLMObsContentBlockType";
7378
export { LLMObsCursorMeta } from "./models/LLMObsCursorMeta";
7479
export { LLMObsCustomEvalConfigAssessmentCriteria } from "./models/LLMObsCustomEvalConfigAssessmentCriteria";
7580
export { LLMObsCustomEvalConfigAttributes } from "./models/LLMObsCustomEvalConfigAttributes";
@@ -132,6 +137,10 @@ export { LLMObsDeleteExperimentsRequest } from "./models/LLMObsDeleteExperiments
132137
export { LLMObsDeleteProjectsDataAttributesRequest } from "./models/LLMObsDeleteProjectsDataAttributesRequest";
133138
export { LLMObsDeleteProjectsDataRequest } from "./models/LLMObsDeleteProjectsDataRequest";
134139
export { LLMObsDeleteProjectsRequest } from "./models/LLMObsDeleteProjectsRequest";
140+
export { LLMObsDisplayBlockAnnotatedInteractionItem } from "./models/LLMObsDisplayBlockAnnotatedInteractionItem";
141+
export { LLMObsDisplayBlockInteractionItem } from "./models/LLMObsDisplayBlockInteractionItem";
142+
export { LLMObsDisplayBlockInteractionResponseItem } from "./models/LLMObsDisplayBlockInteractionResponseItem";
143+
export { LLMObsDisplayBlockInteractionType } from "./models/LLMObsDisplayBlockInteractionType";
135144
export { LLMObsEventType } from "./models/LLMObsEventType";
136145
export { LLMObsExperimentDataAttributesRequest } from "./models/LLMObsExperimentDataAttributesRequest";
137146
export { LLMObsExperimentDataAttributesResponse } from "./models/LLMObsExperimentDataAttributesResponse";
@@ -153,7 +162,6 @@ export { LLMObsExperimentType } from "./models/LLMObsExperimentType";
153162
export { LLMObsExperimentUpdateDataAttributesRequest } from "./models/LLMObsExperimentUpdateDataAttributesRequest";
154163
export { LLMObsExperimentUpdateDataRequest } from "./models/LLMObsExperimentUpdateDataRequest";
155164
export { LLMObsExperimentUpdateRequest } from "./models/LLMObsExperimentUpdateRequest";
156-
export { LLMObsInteractionType } from "./models/LLMObsInteractionType";
157165
export { LLMObsLabelSchema } from "./models/LLMObsLabelSchema";
158166
export { LLMObsLabelSchemaType } from "./models/LLMObsLabelSchemaType";
159167
export { LLMObsMetricAssessment } from "./models/LLMObsMetricAssessment";
@@ -170,3 +178,7 @@ export { LLMObsProjectUpdateDataAttributesRequest } from "./models/LLMObsProject
170178
export { LLMObsProjectUpdateDataRequest } from "./models/LLMObsProjectUpdateDataRequest";
171179
export { LLMObsProjectUpdateRequest } from "./models/LLMObsProjectUpdateRequest";
172180
export { LLMObsRecordType } from "./models/LLMObsRecordType";
181+
export { LLMObsTraceAnnotatedInteractionItem } from "./models/LLMObsTraceAnnotatedInteractionItem";
182+
export { LLMObsTraceInteractionItem } from "./models/LLMObsTraceInteractionItem";
183+
export { LLMObsTraceInteractionResponseItem } from "./models/LLMObsTraceInteractionResponseItem";
184+
export { LLMObsTraceInteractionType } from "./models/LLMObsTraceInteractionType";

‎services/llm_observability/src/v2/models/LLMObsAnnotatedInteractionItem.ts‎

Lines changed: 7 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,12 @@
1-
import { AttributeTypeMap } from "@datadog/datadog-api-client";
1+
import { UnparsedObject } from "@datadog/datadog-api-client";
22

3-
import { LLMObsAnnotationItem } from "./LLMObsAnnotationItem";
4-
import { LLMObsInteractionType } from "./LLMObsInteractionType";
3+
import { LLMObsDisplayBlockAnnotatedInteractionItem } from "./LLMObsDisplayBlockAnnotatedInteractionItem";
4+
import { LLMObsTraceAnnotatedInteractionItem } from "./LLMObsTraceAnnotatedInteractionItem";
55

66
/**
77
* An interaction with its associated annotations.
88
*/
9-
export class LLMObsAnnotatedInteractionItem {
10-
/**
11-
* List of annotations for this interaction.
12-
*/
13-
"annotations": Array<LLMObsAnnotationItem>;
14-
/**
15-
* Identifier of the content (trace ID or session ID) for this interaction.
16-
*/
17-
"contentId": string;
18-
/**
19-
* Unique identifier of the interaction.
20-
*/
21-
"id": string;
22-
/**
23-
* Type of interaction in an annotation queue.
24-
*/
25-
"type": LLMObsInteractionType;
26-
/**
27-
* A container for additional, undeclared properties.
28-
* This is a holder for any undeclared properties as specified with
29-
* the 'additionalProperties' keyword in the OAS document.
30-
*/
31-
"additionalProperties"?: { [key: string]: any };
32-
/**
33-
* @ignore
34-
*/
35-
"_unparsed"?: boolean;
36-
37-
/**
38-
* @ignore
39-
*/
40-
static readonly attributeTypeMap: AttributeTypeMap = {
41-
annotations: {
42-
baseName: "annotations",
43-
type: "Array<LLMObsAnnotationItem>",
44-
required: true,
45-
},
46-
contentId: {
47-
baseName: "content_id",
48-
type: "string",
49-
required: true,
50-
},
51-
id: {
52-
baseName: "id",
53-
type: "string",
54-
required: true,
55-
},
56-
type: {
57-
baseName: "type",
58-
type: "LLMObsInteractionType",
59-
required: true,
60-
},
61-
additionalProperties: {
62-
baseName: "additionalProperties",
63-
type: "{ [key: string]: any; }",
64-
},
65-
};
66-
67-
/**
68-
* @ignore
69-
*/
70-
static getAttributeTypeMap(): AttributeTypeMap {
71-
return LLMObsAnnotatedInteractionItem.attributeTypeMap;
72-
}
73-
74-
public constructor() {}
75-
}
9+
export type LLMObsAnnotatedInteractionItem =
10+
| LLMObsTraceAnnotatedInteractionItem
11+
| LLMObsDisplayBlockAnnotatedInteractionItem
12+
| UnparsedObject;

‎services/llm_observability/src/v2/models/LLMObsAnnotationQueueInteractionItem.ts‎

Lines changed: 7 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,12 @@
1-
import { AttributeTypeMap } from "@datadog/datadog-api-client";
1+
import { UnparsedObject } from "@datadog/datadog-api-client";
22

3-
import { LLMObsInteractionType } from "./LLMObsInteractionType";
3+
import { LLMObsDisplayBlockInteractionItem } from "./LLMObsDisplayBlockInteractionItem";
4+
import { LLMObsTraceInteractionItem } from "./LLMObsTraceInteractionItem";
45

56
/**
67
* A single interaction to add to an annotation queue.
78
*/
8-
export class LLMObsAnnotationQueueInteractionItem {
9-
/**
10-
* Identifier of the content (trace ID or session ID) for this interaction.
11-
*/
12-
"contentId": string;
13-
/**
14-
* Type of interaction in an annotation queue.
15-
*/
16-
"type": LLMObsInteractionType;
17-
/**
18-
* A container for additional, undeclared properties.
19-
* This is a holder for any undeclared properties as specified with
20-
* the 'additionalProperties' keyword in the OAS document.
21-
*/
22-
"additionalProperties"?: { [key: string]: any };
23-
/**
24-
* @ignore
25-
*/
26-
"_unparsed"?: boolean;
27-
28-
/**
29-
* @ignore
30-
*/
31-
static readonly attributeTypeMap: AttributeTypeMap = {
32-
contentId: {
33-
baseName: "content_id",
34-
type: "string",
35-
required: true,
36-
},
37-
type: {
38-
baseName: "type",
39-
type: "LLMObsInteractionType",
40-
required: true,
41-
},
42-
additionalProperties: {
43-
baseName: "additionalProperties",
44-
type: "{ [key: string]: any; }",
45-
},
46-
};
47-
48-
/**
49-
* @ignore
50-
*/
51-
static getAttributeTypeMap(): AttributeTypeMap {
52-
return LLMObsAnnotationQueueInteractionItem.attributeTypeMap;
53-
}
54-
55-
public constructor() {}
56-
}
9+
export type LLMObsAnnotationQueueInteractionItem =
10+
| LLMObsTraceInteractionItem
11+
| LLMObsDisplayBlockInteractionItem
12+
| UnparsedObject;

‎services/llm_observability/src/v2/models/LLMObsAnnotationQueueInteractionResponseItem.ts‎

Lines changed: 7 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,12 @@
1-
import { AttributeTypeMap } from "@datadog/datadog-api-client";
1+
import { UnparsedObject } from "@datadog/datadog-api-client";
22

3-
import { LLMObsInteractionType } from "./LLMObsInteractionType";
3+
import { LLMObsDisplayBlockInteractionResponseItem } from "./LLMObsDisplayBlockInteractionResponseItem";
4+
import { LLMObsTraceInteractionResponseItem } from "./LLMObsTraceInteractionResponseItem";
45

56
/**
67
* A single interaction result.
78
*/
8-
export class LLMObsAnnotationQueueInteractionResponseItem {
9-
/**
10-
* Whether this interaction already existed in the queue.
11-
*/
12-
"alreadyExisted": boolean;
13-
/**
14-
* Identifier of the content (trace ID or session ID) for this interaction.
15-
*/
16-
"contentId": string;
17-
/**
18-
* Unique identifier of the interaction.
19-
*/
20-
"id": string;
21-
/**
22-
* Type of interaction in an annotation queue.
23-
*/
24-
"type": LLMObsInteractionType;
25-
/**
26-
* A container for additional, undeclared properties.
27-
* This is a holder for any undeclared properties as specified with
28-
* the 'additionalProperties' keyword in the OAS document.
29-
*/
30-
"additionalProperties"?: { [key: string]: any };
31-
/**
32-
* @ignore
33-
*/
34-
"_unparsed"?: boolean;
35-
36-
/**
37-
* @ignore
38-
*/
39-
static readonly attributeTypeMap: AttributeTypeMap = {
40-
alreadyExisted: {
41-
baseName: "already_existed",
42-
type: "boolean",
43-
required: true,
44-
},
45-
contentId: {
46-
baseName: "content_id",
47-
type: "string",
48-
required: true,
49-
},
50-
id: {
51-
baseName: "id",
52-
type: "string",
53-
required: true,
54-
},
55-
type: {
56-
baseName: "type",
57-
type: "LLMObsInteractionType",
58-
required: true,
59-
},
60-
additionalProperties: {
61-
baseName: "additionalProperties",
62-
type: "{ [key: string]: any; }",
63-
},
64-
};
65-
66-
/**
67-
* @ignore
68-
*/
69-
static getAttributeTypeMap(): AttributeTypeMap {
70-
return LLMObsAnnotationQueueInteractionResponseItem.attributeTypeMap;
71-
}
72-
73-
public constructor() {}
74-
}
9+
export type LLMObsAnnotationQueueInteractionResponseItem =
10+
| LLMObsTraceInteractionResponseItem
11+
| LLMObsDisplayBlockInteractionResponseItem
12+
| UnparsedObject;

0 commit comments

Comments
 (0)