Skip to content

Commit 1139196

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for LLM Observability eval config CRUD endpoints (#3744)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent b3f8403 commit 1139196

31 files changed

+6106
-24
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 651 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Delete a custom evaluator configuration returns "No Content" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
7+
public class Example {
8+
public static void main(String[] args) {
9+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
10+
defaultClient.setUnstableOperationEnabled("v2.deleteLLMObsCustomEvalConfig", true);
11+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
12+
13+
try {
14+
apiInstance.deleteLLMObsCustomEvalConfig("my-custom-evaluator");
15+
} catch (ApiException e) {
16+
System.err.println("Exception when calling LlmObservabilityApi#deleteLLMObsCustomEvalConfig");
17+
System.err.println("Status code: " + e.getCode());
18+
System.err.println("Reason: " + e.getResponseBody());
19+
System.err.println("Response headers: " + e.getResponseHeaders());
20+
e.printStackTrace();
21+
}
22+
}
23+
}

‎examples/v2/llm-observability/GetLLMObsCustomEvalConfig.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Get a custom evaluator configuration returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigResponse;
7+
8+
public class Example {
9+
public static void main(String[] args) {
10+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
11+
defaultClient.setUnstableOperationEnabled("v2.getLLMObsCustomEvalConfig", true);
12+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
13+
14+
try {
15+
LLMObsCustomEvalConfigResponse result =
16+
apiInstance.getLLMObsCustomEvalConfig("my-custom-evaluator");
17+
System.out.println(result);
18+
} catch (ApiException e) {
19+
System.err.println("Exception when calling LlmObservabilityApi#getLLMObsCustomEvalConfig");
20+
System.err.println("Status code: " + e.getCode());
21+
System.err.println("Reason: " + e.getResponseBody());
22+
System.err.println("Response headers: " + e.getResponseHeaders());
23+
e.printStackTrace();
24+
}
25+
}
26+
}

‎examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.java‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Create or update a custom evaluator configuration returns "OK" response
2+
3+
import com.datadog.api.client.ApiClient;
4+
import com.datadog.api.client.ApiException;
5+
import com.datadog.api.client.v2.api.LlmObservabilityApi;
6+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigAssessmentCriteria;
7+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigBedrockOptions;
8+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigEvalScope;
9+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigInferenceParams;
10+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigIntegrationProvider;
11+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigLLMJudgeConfig;
12+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigLLMProvider;
13+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigParsingType;
14+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigPromptContent;
15+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigPromptContentValue;
16+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigPromptMessage;
17+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigPromptToolCall;
18+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigPromptToolResult;
19+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigTarget;
20+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigType;
21+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigUpdateAttributes;
22+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigUpdateData;
23+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigUpdateRequest;
24+
import com.datadog.api.client.v2.model.LLMObsCustomEvalConfigVertexAIOptions;
25+
import java.util.Arrays;
26+
import java.util.Collections;
27+
28+
public class Example {
29+
public static void main(String[] args) {
30+
ApiClient defaultClient = ApiClient.getDefaultApiClient();
31+
defaultClient.setUnstableOperationEnabled("v2.updateLLMObsCustomEvalConfig", true);
32+
LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient);
33+
34+
LLMObsCustomEvalConfigUpdateRequest body =
35+
new LLMObsCustomEvalConfigUpdateRequest()
36+
.data(
37+
new LLMObsCustomEvalConfigUpdateData()
38+
.attributes(
39+
new LLMObsCustomEvalConfigUpdateAttributes()
40+
.category("Custom")
41+
.evalName("my-custom-evaluator")
42+
.llmJudgeConfig(
43+
new LLMObsCustomEvalConfigLLMJudgeConfig()
44+
.assessmentCriteria(
45+
new LLMObsCustomEvalConfigAssessmentCriteria()
46+
.maxThreshold(1.0)
47+
.minThreshold(0.7)
48+
.passValues(Arrays.asList("pass", "yes"))
49+
.passWhen(true))
50+
.inferenceParams(
51+
new LLMObsCustomEvalConfigInferenceParams()
52+
.maxTokens(1024L)
53+
.temperature(0.7)
54+
.topK(50L)
55+
.topP(1.0))
56+
.lastUsedLibraryPromptTemplateName("sentiment-analysis-v1")
57+
.modifiedLibraryPromptTemplate(false)
58+
.outputSchema(null)
59+
.parsingType(
60+
LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT)
61+
.promptTemplate(
62+
Collections.singletonList(
63+
new LLMObsCustomEvalConfigPromptMessage()
64+
.content(
65+
"Rate the quality of the following response:")
66+
.contents(
67+
Collections.singletonList(
68+
new LLMObsCustomEvalConfigPromptContent()
69+
.type("text")
70+
.value(
71+
new LLMObsCustomEvalConfigPromptContentValue()
72+
.text(
73+
"What is the sentiment of"
74+
+ " this review?")
75+
.toolCall(
76+
new LLMObsCustomEvalConfigPromptToolCall()
77+
.arguments(
78+
"""
79+
{"location": "San Francisco"}
80+
""")
81+
.id("call_abc123")
82+
.name("get_weather")
83+
.type("function"))
84+
.toolCallResult(
85+
new LLMObsCustomEvalConfigPromptToolResult()
86+
.name("get_weather")
87+
.result("sunny, 72F")
88+
.toolId("call_abc123")
89+
.type("function")))))
90+
.role("user"))))
91+
.llmProvider(
92+
new LLMObsCustomEvalConfigLLMProvider()
93+
.bedrock(
94+
new LLMObsCustomEvalConfigBedrockOptions()
95+
.region("us-east-1"))
96+
.integrationAccountId("my-account-id")
97+
.integrationProvider(
98+
LLMObsCustomEvalConfigIntegrationProvider.OPENAI)
99+
.modelName("gpt-4o")
100+
.vertexAi(
101+
new LLMObsCustomEvalConfigVertexAIOptions()
102+
.location("us-central1")
103+
.project("my-gcp-project")))
104+
.target(
105+
new LLMObsCustomEvalConfigTarget()
106+
.applicationName("my-llm-app")
107+
.enabled(true)
108+
.evalScope(LLMObsCustomEvalConfigEvalScope.SPAN)
109+
.filter("@service:my-service")
110+
.rootSpansOnly(true)
111+
.samplingPercentage(50.0)))
112+
.id("my-custom-evaluator")
113+
.type(LLMObsCustomEvalConfigType.EVALUATOR_CONFIG));
114+
115+
try {
116+
apiInstance.updateLLMObsCustomEvalConfig("my-custom-evaluator", body);
117+
} catch (ApiException e) {
118+
System.err.println("Exception when calling LlmObservabilityApi#updateLLMObsCustomEvalConfig");
119+
System.err.println("Status code: " + e.getCode());
120+
System.err.println("Reason: " + e.getResponseBody());
121+
System.err.println("Response headers: " + e.getResponseHeaders());
122+
e.printStackTrace();
123+
}
124+
}
125+
}

‎src/main/java/com/datadog/api/client/ApiClient.java‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -768,6 +768,33 @@ public class ApiClient {
768768
put("v2.listFleetTracers", false);
769769
put("v2.triggerFleetSchedule", false);
770770
put("v2.updateFleetSchedule", false);
771+
put("v2.createLLMObsAnnotationQueue", false);
772+
put("v2.createLLMObsAnnotationQueueInteractions", false);
773+
put("v2.createLLMObsDataset", false);
774+
put("v2.createLLMObsDatasetRecords", false);
775+
put("v2.createLLMObsExperiment", false);
776+
put("v2.createLLMObsExperimentEvents", false);
777+
put("v2.createLLMObsProject", false);
778+
put("v2.deleteLLMObsAnnotationQueue", false);
779+
put("v2.deleteLLMObsAnnotationQueueInteractions", false);
780+
put("v2.deleteLLMObsCustomEvalConfig", false);
781+
put("v2.deleteLLMObsDatasetRecords", false);
782+
put("v2.deleteLLMObsDatasets", false);
783+
put("v2.deleteLLMObsExperiments", false);
784+
put("v2.deleteLLMObsProjects", false);
785+
put("v2.getLLMObsAnnotatedInteractions", false);
786+
put("v2.getLLMObsCustomEvalConfig", false);
787+
put("v2.listLLMObsAnnotationQueues", false);
788+
put("v2.listLLMObsDatasetRecords", false);
789+
put("v2.listLLMObsDatasets", false);
790+
put("v2.listLLMObsExperiments", false);
791+
put("v2.listLLMObsProjects", false);
792+
put("v2.updateLLMObsAnnotationQueue", false);
793+
put("v2.updateLLMObsCustomEvalConfig", false);
794+
put("v2.updateLLMObsDataset", false);
795+
put("v2.updateLLMObsDatasetRecords", false);
796+
put("v2.updateLLMObsExperiment", false);
797+
put("v2.updateLLMObsProject", false);
771798
put("v2.createOpenAPI", false);
772799
put("v2.deleteOpenAPI", false);
773800
put("v2.getOpenAPI", false);
@@ -921,30 +948,6 @@ public class ApiClient {
921948
put("v2.updateJiraIssueTemplate", false);
922949
put("v2.createTenancyConfig", false);
923950
put("v2.getTenancyConfigs", false);
924-
put("v2.createLLMObsAnnotationQueue", false);
925-
put("v2.createLLMObsAnnotationQueueInteractions", false);
926-
put("v2.createLLMObsDataset", false);
927-
put("v2.createLLMObsDatasetRecords", false);
928-
put("v2.createLLMObsExperiment", false);
929-
put("v2.createLLMObsExperimentEvents", false);
930-
put("v2.createLLMObsProject", false);
931-
put("v2.deleteLLMObsAnnotationQueue", false);
932-
put("v2.deleteLLMObsAnnotationQueueInteractions", false);
933-
put("v2.deleteLLMObsDatasetRecords", false);
934-
put("v2.deleteLLMObsDatasets", false);
935-
put("v2.deleteLLMObsExperiments", false);
936-
put("v2.deleteLLMObsProjects", false);
937-
put("v2.getLLMObsAnnotatedInteractions", false);
938-
put("v2.listLLMObsAnnotationQueues", false);
939-
put("v2.listLLMObsDatasetRecords", false);
940-
put("v2.listLLMObsDatasets", false);
941-
put("v2.listLLMObsExperiments", false);
942-
put("v2.listLLMObsProjects", false);
943-
put("v2.updateLLMObsAnnotationQueue", false);
944-
put("v2.updateLLMObsDataset", false);
945-
put("v2.updateLLMObsDatasetRecords", false);
946-
put("v2.updateLLMObsExperiment", false);
947-
put("v2.updateLLMObsProject", false);
948951
put("v2.addRoleToRestrictionQuery", false);
949952
put("v2.createRestrictionQuery", false);
950953
put("v2.deleteRestrictionQuery", false);

0 commit comments

Comments
 (0)