Add OpenAPI spec for LLM Observability Experiments API (#3553)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -0,0 +1,30 @@
+/**
+ * Create an LLM Observability dataset returns "OK" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.createLLMObsDataset"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiCreateLLMObsDatasetRequest = {
+  body: {
+    data: {
+      attributes: {
+        name: "My LLM Dataset",
+      },
+      type: "datasets",
+    },
+  },
+  projectId: "project_id",
+};
+
+apiInstance
+  .createLLMObsDataset(params)
+  .then((data: v2.LLMObsDatasetResponse) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/CreateLLMObsDataset.ts

@@ -0,0 +1,36 @@
+/**
+ * Append records to an LLM Observability dataset returns "OK" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.createLLMObsDatasetRecords"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiCreateLLMObsDatasetRecordsRequest = {
+  body: {
+    data: {
+      attributes: {
+        records: [
+          {
+            expectedOutput: undefined,
+            input: null,
+          },
+        ],
+      },
+      type: "records",
+    },
+  },
+  projectId: "project_id",
+  datasetId: "dataset_id",
+};
+
+apiInstance
+  .createLLMObsDatasetRecords(params)
+  .then((data: v2.LLMObsDatasetRecordsMutationResponse) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/CreateLLMObsDatasetRecords.ts

@@ -0,0 +1,31 @@
+/**
+ * Create an LLM Observability experiment returns "OK" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.createLLMObsExperiment"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiCreateLLMObsExperimentRequest = {
+  body: {
+    data: {
+      attributes: {
+        datasetId: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d",
+        name: "My Experiment v1",
+        projectId: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751",
+      },
+      type: "experiments",
+    },
+  },
+};
+
+apiInstance
+  .createLLMObsExperiment(params)
+  .then((data: v2.LLMObsExperimentResponse) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/CreateLLMObsExperiment.ts

@@ -0,0 +1,67 @@
+/**
+ * Push events for an LLM Observability experiment returns "Accepted" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.createLLMObsExperimentEvents"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiCreateLLMObsExperimentEventsRequest = {
+  body: {
+    data: {
+      attributes: {
+        metrics: [
+          {
+            assessment: "pass",
+            error: {},
+            label: "faithfulness",
+            metricType: "score",
+            spanId: "span-7a1b2c3d",
+            tags: [],
+            timestampMs: 1705314600000,
+          },
+        ],
+        spans: [
+          {
+            datasetId: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d",
+            duration: 1500000000,
+            meta: {
+              error: {
+                message: "Model response timed out",
+                stack: `Traceback (most recent call last):
+  File "main.py", line 10, in <module>
+    response = model.generate(input)
+  File "model.py", line 45, in generate
+    raise TimeoutError("Model response timed out")
+TimeoutError: Model response timed out`,
+                type: "TimeoutError",
+              },
+              input: undefined,
+              output: undefined,
+            },
+            name: "llm_call",
+            projectId: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751",
+            spanId: "span-7a1b2c3d",
+            startNs: 1705314600000000000,
+            status: "ok",
+            tags: [],
+            traceId: "abc123def456",
+          },
+        ],
+      },
+      type: "events",
+    },
+  },
+  experimentId: "experiment_id",
+};
+
+apiInstance
+  .createLLMObsExperimentEvents(params)
+  .then((data: any) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/CreateLLMObsExperimentEvents.ts

@@ -0,0 +1,29 @@
+/**
+ * Create an LLM Observability project returns "OK" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.createLLMObsProject"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiCreateLLMObsProjectRequest = {
+  body: {
+    data: {
+      attributes: {
+        name: "My LLM Project",
+      },
+      type: "projects",
+    },
+  },
+};
+
+apiInstance
+  .createLLMObsProject(params)
+  .then((data: v2.LLMObsProjectResponse) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/CreateLLMObsProject.ts

@@ -0,0 +1,31 @@
+/**
+ * Delete LLM Observability dataset records returns "No Content" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.deleteLLMObsDatasetRecords"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiDeleteLLMObsDatasetRecordsRequest = {
+  body: {
+    data: {
+      attributes: {
+        recordIds: ["rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c"],
+      },
+      type: "records",
+    },
+  },
+  projectId: "project_id",
+  datasetId: "dataset_id",
+};
+
+apiInstance
+  .deleteLLMObsDatasetRecords(params)
+  .then((data: any) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/DeleteLLMObsDatasetRecords.ts

@@ -0,0 +1,30 @@
+/**
+ * Delete LLM Observability datasets returns "No Content" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.deleteLLMObsDatasets"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiDeleteLLMObsDatasetsRequest = {
+  body: {
+    data: {
+      attributes: {
+        datasetIds: ["9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d"],
+      },
+      type: "datasets",
+    },
+  },
+  projectId: "project_id",
+};
+
+apiInstance
+  .deleteLLMObsDatasets(params)
+  .then((data: any) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/DeleteLLMObsDatasets.ts

@@ -0,0 +1,29 @@
+/**
+ * Delete LLM Observability experiments returns "No Content" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.deleteLLMObsExperiments"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiDeleteLLMObsExperimentsRequest = {
+  body: {
+    data: {
+      attributes: {
+        experimentIds: ["3fd6b5e0-8910-4b1c-a7d0-5b84de329012"],
+      },
+      type: "experiments",
+    },
+  },
+};
+
+apiInstance
+  .deleteLLMObsExperiments(params)
+  .then((data: any) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

examples/v2/llm-observability/DeleteLLMObsExperiments.ts

@@ -0,0 +1,29 @@
+/**
+ * Delete LLM Observability projects returns "No Content" response
+ */
+
+import { client, v2 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+configuration.unstableOperations["v2.deleteLLMObsProjects"] = true;
+const apiInstance = new v2.LLMObservabilityApi(configuration);
+
+const params: v2.LLMObservabilityApiDeleteLLMObsProjectsRequest = {
+  body: {
+    data: {
+      attributes: {
+        projectIds: ["a33671aa-24fd-4dcd-9b33-a8ec7dde7751"],
+      },
+      type: "projects",
+    },
+  },
+};
+
+apiInstance
+  .deleteLLMObsProjects(params)
+  .then((data: any) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));