Skip to content

Commit 9c2aecb

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI spec for LLM Observability Experiments API (#3537)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent aaf315d commit 9c2aecb

File tree

93 files changed

+20026
-677
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

93 files changed

+20026
-677
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 2672 additions & 677 deletions
Large diffs are not rendered by default.

‎examples/v2/llm-observability/CreateLLMObsDataset.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Create an LLM Observability dataset returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsDatasetDataAttributesRequest;
7+
import com.datadog.api.client.v2.model.LLMObsDatasetDataRequest;
8+
import com.datadog.api.client.v2.model.LLMObsDatasetRequest;
9+
import com.datadog.api.client.v2.model.LLMObsDatasetResponse;
10+
import com.datadog.api.client.v2.model.LLMObsDatasetType;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.createLLMObsDataset", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsDatasetRequest body =
19+
new LLMObsDatasetRequest()
20+
.data(
21+
new LLMObsDatasetDataRequest()
22+
.attributes(new LLMObsDatasetDataAttributesRequest().name("My LLM Dataset"))
23+
.type(LLMObsDatasetType.DATASETS));
24+
25+
try {
26+
LLMObsDatasetResponse result =
27+
apiInstance.createLLMObsDataset("a33671aa-24fd-4dcd-9b33-a8ec7dde7751", body);
28+
System.out.println(result);
29+
} catch (ApiException e) {
30+
System.err.println("Exception when calling LlmObservabilityApi#createLLMObsDataset");
31+
System.err.println("Status code: " + e.getCode());
32+
System.err.println("Reason: " + e.getResponseBody());
33+
System.err.println("Response headers: " + e.getResponseHeaders());
34+
e.printStackTrace();
35+
}
36+
}
37+
}

‎examples/v2/llm-observability/CreateLLMObsDatasetRecords.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Append records to an LLM Observability dataset returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsDatasetRecordItem;
7+
import com.datadog.api.client.v2.model.LLMObsDatasetRecordsDataAttributesRequest;
8+
import com.datadog.api.client.v2.model.LLMObsDatasetRecordsDataRequest;
9+
import com.datadog.api.client.v2.model.LLMObsDatasetRecordsMutationResponse;
10+
import com.datadog.api.client.v2.model.LLMObsDatasetRecordsRequest;
11+
import com.datadog.api.client.v2.model.LLMObsRecordType;
12+
import java.util.Collections;
13+
14+
public class Example {
15+
public static void main(String[] args) {
16+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
17+
defaultClient.setUnstableOperationEnabled("v2.createLLMObsDatasetRecords", true);
18+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
19+
20+
LLMObsDatasetRecordsRequest body =
21+
new LLMObsDatasetRecordsRequest()
22+
.data(
23+
new LLMObsDatasetRecordsDataRequest()
24+
.attributes(
25+
new LLMObsDatasetRecordsDataAttributesRequest()
26+
.records(
27+
Collections.singletonList(
28+
new LLMObsDatasetRecordItem()
29+
.expectedOutput(null)
30+
.input(null))))
31+
.type(LLMObsRecordType.RECORDS));
32+
33+
try {
34+
LLMObsDatasetRecordsMutationResponse result =
35+
apiInstance.createLLMObsDatasetRecords(
36+
"a33671aa-24fd-4dcd-9b33-a8ec7dde7751", "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", body);
37+
System.out.println(result);
38+
} catch (ApiException e) {
39+
System.err.println("Exception when calling LlmObservabilityApi#createLLMObsDatasetRecords");
40+
System.err.println("Status code: " + e.getCode());
41+
System.err.println("Reason: " + e.getResponseBody());
42+
System.err.println("Response headers: " + e.getResponseHeaders());
43+
e.printStackTrace();
44+
}
45+
}
46+
}

‎examples/v2/llm-observability/CreateLLMObsExperiment.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Create an LLM Observability experiment returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsExperimentDataAttributesRequest;
7+
import com.datadog.api.client.v2.model.LLMObsExperimentDataRequest;
8+
import com.datadog.api.client.v2.model.LLMObsExperimentRequest;
9+
import com.datadog.api.client.v2.model.LLMObsExperimentResponse;
10+
import com.datadog.api.client.v2.model.LLMObsExperimentType;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.createLLMObsExperiment", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsExperimentRequest body =
19+
new LLMObsExperimentRequest()
20+
.data(
21+
new LLMObsExperimentDataRequest()
22+
.attributes(
23+
new LLMObsExperimentDataAttributesRequest()
24+
.datasetId("9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d")
25+
.name("My Experiment v1")
26+
.projectId("a33671aa-24fd-4dcd-9b33-a8ec7dde7751"))
27+
.type(LLMObsExperimentType.EXPERIMENTS));
28+
29+
try {
30+
LLMObsExperimentResponse result = apiInstance.createLLMObsExperiment(body);
31+
System.out.println(result);
32+
} catch (ApiException e) {
33+
System.err.println("Exception when calling LlmObservabilityApi#createLLMObsExperiment");
34+
System.err.println("Status code: " + e.getCode());
35+
System.err.println("Reason: " + e.getResponseBody());
36+
System.err.println("Response headers: " + e.getResponseHeaders());
37+
e.printStackTrace();
38+
}
39+
}
40+
}

‎examples/v2/llm-observability/CreateLLMObsExperimentEvents.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Push events for an LLM Observability experiment returns "Accepted" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsEventType;
7+
import com.datadog.api.client.v2.model.LLMObsExperimentEventsDataAttributesRequest;
8+
import com.datadog.api.client.v2.model.LLMObsExperimentEventsDataRequest;
9+
import com.datadog.api.client.v2.model.LLMObsExperimentEventsRequest;
10+
import com.datadog.api.client.v2.model.LLMObsExperimentMetric;
11+
import com.datadog.api.client.v2.model.LLMObsExperimentMetricError;
12+
import com.datadog.api.client.v2.model.LLMObsExperimentSpan;
13+
import com.datadog.api.client.v2.model.LLMObsExperimentSpanError;
14+
import com.datadog.api.client.v2.model.LLMObsExperimentSpanMeta;
15+
import com.datadog.api.client.v2.model.LLMObsExperimentSpanStatus;
16+
import com.datadog.api.client.v2.model.LLMObsMetricAssessment;
17+
import com.datadog.api.client.v2.model.LLMObsMetricScoreType;
18+
import java.util.Collections;
19+
20+
public class Example {
21+
public static void main(String[] args) {
22+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
23+
defaultClient.setUnstableOperationEnabled("v2.createLLMObsExperimentEvents", true);
24+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
25+
26+
LLMObsExperimentEventsRequest body =
27+
new LLMObsExperimentEventsRequest()
28+
.data(
29+
new LLMObsExperimentEventsDataRequest()
30+
.attributes(
31+
new LLMObsExperimentEventsDataAttributesRequest()
32+
.metrics(
33+
Collections.singletonList(
34+
new LLMObsExperimentMetric()
35+
.assessment(LLMObsMetricAssessment.PASS)
36+
.error(new LLMObsExperimentMetricError())
37+
.label("faithfulness")
38+
.metricType(LLMObsMetricScoreType.SCORE)
39+
.spanId("span-7a1b2c3d")
40+
.timestampMs(1705314600000L)))
41+
.spans(
42+
Collections.singletonList(
43+
new LLMObsExperimentSpan()
44+
.datasetId("9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d")
45+
.duration(1500000000L)
46+
.meta(
47+
new LLMObsExperimentSpanMeta()
48+
.error(
49+
new LLMObsExperimentSpanError()
50+
.message("Model response timed out")
51+
.stack(
52+
"""
53+
Traceback (most recent call last):
54+
File "main.py", line 10, in <module>
55+
response = model.generate(input)
56+
File "model.py", line 45, in generate
57+
raise TimeoutError("Model response timed out")
58+
TimeoutError: Model response timed out
59+
""")
60+
.type("TimeoutError"))
61+
.input(null)
62+
.output(null))
63+
.name("llm_call")
64+
.projectId("a33671aa-24fd-4dcd-9b33-a8ec7dde7751")
65+
.spanId("span-7a1b2c3d")
66+
.startNs(1705314600000000000L)
67+
.status(LLMObsExperimentSpanStatus.OK)
68+
.traceId("abc123def456"))))
69+
.type(LLMObsEventType.EVENTS));
70+
71+
try {
72+
apiInstance.createLLMObsExperimentEvents("3fd6b5e0-8910-4b1c-a7d0-5b84de329012", body);
73+
} catch (ApiException e) {
74+
System.err.println("Exception when calling LlmObservabilityApi#createLLMObsExperimentEvents");
75+
System.err.println("Status code: " + e.getCode());
76+
System.err.println("Reason: " + e.getResponseBody());
77+
System.err.println("Response headers: " + e.getResponseHeaders());
78+
e.printStackTrace();
79+
}
80+
}
81+
}

‎examples/v2/llm-observability/CreateLLMObsProject.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Create an LLM Observability project returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsProjectDataAttributesRequest;
7+
import com.datadog.api.client.v2.model.LLMObsProjectDataRequest;
8+
import com.datadog.api.client.v2.model.LLMObsProjectRequest;
9+
import com.datadog.api.client.v2.model.LLMObsProjectResponse;
10+
import com.datadog.api.client.v2.model.LLMObsProjectType;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.createLLMObsProject", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsProjectRequest body =
19+
new LLMObsProjectRequest()
20+
.data(
21+
new LLMObsProjectDataRequest()
22+
.attributes(new LLMObsProjectDataAttributesRequest().name("My LLM Project"))
23+
.type(LLMObsProjectType.PROJECTS));
24+
25+
try {
26+
LLMObsProjectResponse result = apiInstance.createLLMObsProject(body);
27+
System.out.println(result);
28+
} catch (ApiException e) {
29+
System.err.println("Exception when calling LlmObservabilityApi#createLLMObsProject");
30+
System.err.println("Status code: " + e.getCode());
31+
System.err.println("Reason: " + e.getResponseBody());
32+
System.err.println("Response headers: " + e.getResponseHeaders());
33+
e.printStackTrace();
34+
}
35+
}
36+
}

‎examples/v2/llm-observability/DeleteLLMObsDatasetRecords.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Delete LLM Observability dataset records returns "No Content" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetRecordsDataAttributesRequest;
7+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetRecordsDataRequest;
8+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetRecordsRequest;
9+
import com.datadog.api.client.v2.model.LLMObsRecordType;
10+
import java.util.Collections;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.deleteLLMObsDatasetRecords", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsDeleteDatasetRecordsRequest body =
19+
new LLMObsDeleteDatasetRecordsRequest()
20+
.data(
21+
new LLMObsDeleteDatasetRecordsDataRequest()
22+
.attributes(
23+
new LLMObsDeleteDatasetRecordsDataAttributesRequest()
24+
.recordIds(
25+
Collections.singletonList(
26+
"rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c")))
27+
.type(LLMObsRecordType.RECORDS));
28+
29+
try {
30+
apiInstance.deleteLLMObsDatasetRecords(
31+
"a33671aa-24fd-4dcd-9b33-a8ec7dde7751", "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", body);
32+
} catch (ApiException e) {
33+
System.err.println("Exception when calling LlmObservabilityApi#deleteLLMObsDatasetRecords");
34+
System.err.println("Status code: " + e.getCode());
35+
System.err.println("Reason: " + e.getResponseBody());
36+
System.err.println("Response headers: " + e.getResponseHeaders());
37+
e.printStackTrace();
38+
}
39+
}
40+
}

‎examples/v2/llm-observability/DeleteLLMObsDatasets.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Delete LLM Observability datasets returns "No Content" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsDatasetType;
7+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetsDataAttributesRequest;
8+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetsDataRequest;
9+
import com.datadog.api.client.v2.model.LLMObsDeleteDatasetsRequest;
10+
import java.util.Collections;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.deleteLLMObsDatasets", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsDeleteDatasetsRequest body =
19+
new LLMObsDeleteDatasetsRequest()
20+
.data(
21+
new LLMObsDeleteDatasetsDataRequest()
22+
.attributes(
23+
new LLMObsDeleteDatasetsDataAttributesRequest()
24+
.datasetIds(
25+
Collections.singletonList("9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d")))
26+
.type(LLMObsDatasetType.DATASETS));
27+
28+
try {
29+
apiInstance.deleteLLMObsDatasets("a33671aa-24fd-4dcd-9b33-a8ec7dde7751", body);
30+
} catch (ApiException e) {
31+
System.err.println("Exception when calling LlmObservabilityApi#deleteLLMObsDatasets");
32+
System.err.println("Status code: " + e.getCode());
33+
System.err.println("Reason: " + e.getResponseBody());
34+
System.err.println("Response headers: " + e.getResponseHeaders());
35+
e.printStackTrace();
36+
}
37+
}
38+
}

‎examples/v2/llm-observability/DeleteLLMObsExperiments.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// Delete LLM Observability experiments returns "No Content" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsDeleteExperimentsDataAttributesRequest;
7+
import com.datadog.api.client.v2.model.LLMObsDeleteExperimentsDataRequest;
8+
import com.datadog.api.client.v2.model.LLMObsDeleteExperimentsRequest;
9+
import com.datadog.api.client.v2.model.LLMObsExperimentType;
10+
import java.util.Collections;
11+
12+
public class Example {
13+
public static void main(String[] args) {
14+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
15+
defaultClient.setUnstableOperationEnabled("v2.deleteLLMObsExperiments", true);
16+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
17+
18+
LLMObsDeleteExperimentsRequest body =
19+
new LLMObsDeleteExperimentsRequest()
20+
.data(
21+
new LLMObsDeleteExperimentsDataRequest()
22+
.attributes(
23+
new LLMObsDeleteExperimentsDataAttributesRequest()
24+
.experimentIds(
25+
Collections.singletonList("3fd6b5e0-8910-4b1c-a7d0-5b84de329012")))
26+
.type(LLMObsExperimentType.EXPERIMENTS));
27+
28+
try {
29+
apiInstance.deleteLLMObsExperiments(body);
30+
} catch (ApiException e) {
31+
System.err.println("Exception when calling LlmObservabilityApi#deleteLLMObsExperiments");
32+
System.err.println("Status code: " + e.getCode());
33+
System.err.println("Reason: " + e.getResponseBody());
34+
System.err.println("Response headers: " + e.getResponseHeaders());
35+
e.printStackTrace();
36+
}
37+
}
38+
}

0 commit comments

Comments
 (0)