Add Obs Pipelines Elasticsearch metrics destination and new fields (#3864)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -43722,6 +43722,7 @@ components:
     ObservabilityPipelineConfigDestinationItem:
       description: "A destination for the pipeline."
       oneOf:
+        - $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineAmazonOpenSearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineAmazonS3Destination"
@@ -43731,7 +43732,6 @@ components:
         - $ref: "#/components/schemas/ObservabilityPipelineCloudPremDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineDatadogLogsDestination"
-        - $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGoogleChronicleDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGoogleCloudStorageDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGooglePubSubDestination"
@@ -44333,9 +44333,9 @@ components:
       type: object
     ObservabilityPipelineElasticsearchDestination:
       description: |-
-        The `elasticsearch` destination writes logs to an Elasticsearch cluster.
+        The `elasticsearch` destination writes logs or metrics to an Elasticsearch cluster.
 
-        **Supported pipeline types:** logs
+        **Supported pipeline types:** logs, metrics
       properties:
         api_version:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationApiVersion"
@@ -44344,9 +44344,11 @@ components:
         buffer:
           $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         bulk_index:
-          description: The index to write logs to in Elasticsearch.
+          description: The name of the index to write events to in Elasticsearch.
           example: logs-index
           type: string
+        compression:
+          $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationCompression"
         data_stream:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationDataStream"
         endpoint_url_key:
@@ -44357,21 +44359,34 @@ components:
           description: The unique identifier for this component.
           example: "elasticsearch-destination"
           type: string
+        id_key:
+          description: The name of the field used as the document ID in Elasticsearch.
+          example: id
+          type: string
         inputs:
           description: A list of component IDs whose output is used as the `input` for this component.
           example: ["filter-processor"]
           items:
             description: The ID of a component whose output is used as input for this destination.
             type: string
           type: array
+        pipeline:
+          description: The name of an Elasticsearch ingest pipeline to apply to events before indexing.
+          example: my-pipeline
+          type: string
+        request_retry_partial:
+          description: When `true`, retries failed partial bulk requests when some events in a batch fail while others succeed.
+          type: boolean
+        tls:
+          $ref: "#/components/schemas/ObservabilityPipelineTls"
         type:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationType"
       required:
         - id
         - type
         - inputs
       type: object
-      x-pipeline-types: [logs]
+      x-pipeline-types: [logs, metrics]
     ObservabilityPipelineElasticsearchDestinationApiVersion:
       description: The Elasticsearch API version to use. Set to `auto` to auto-detect.
       enum: [auto, v6, v7, v8]
@@ -44400,18 +44415,48 @@ components:
       required:
         - strategy
       type: object
+    ObservabilityPipelineElasticsearchDestinationCompression:
+      description: Compression configuration for the Elasticsearch destination.
+      properties:
+        algorithm:
+          $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm"
+        level:
+          description: The compression level. Only applicable for `gzip`, `zlib`, and `zstd` algorithms.
+          example: 6
+          format: int64
+          type: integer
+      required:
+        - algorithm
+      type: object
+    ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm:
+      description: The compression algorithm applied when sending data to Elasticsearch.
+      enum: [none, gzip, zlib, zstd, snappy]
+      example: gzip
+      type: string
+      x-enum-varnames:
+        - NONE
+        - GZIP
+        - ZLIB
+        - ZSTD
+        - SNAPPY
     ObservabilityPipelineElasticsearchDestinationDataStream:
       description: Configuration options for writing to Elasticsearch Data Streams instead of a fixed index.
       properties:
+        auto_routing:
+          description: When `true`, automatically routes events to the appropriate data stream based on the event content.
+          type: boolean
         dataset:
-          description: The data stream dataset for your logs. This groups logs by their source or application.
+          description: The data stream dataset. This groups events by their source or application.
           type: string
         dtype:
-          description: The data stream type for your logs. This determines how logs are categorized within the data stream.
+          description: The data stream type. This determines how events are categorized within the data stream.
           type: string
         namespace:
-          description: The data stream namespace for your logs. This separates logs into different environments or domains.
+          description: The data stream namespace. This separates events into different environments or domains.
           type: string
+        sync_fields:
+          description: When `true`, synchronizes data stream fields with the Elasticsearch index mapping.
+          type: boolean
       type: object
     ObservabilityPipelineElasticsearchDestinationType:
       default: elasticsearch

services/observability_pipelines/src/v2/index.ts

@@ -95,6 +95,8 @@ export { ObservabilityPipelineDiskBufferOptions } from "./models/ObservabilityPi
 export { ObservabilityPipelineElasticsearchDestination } from "./models/ObservabilityPipelineElasticsearchDestination";
 export { ObservabilityPipelineElasticsearchDestinationApiVersion } from "./models/ObservabilityPipelineElasticsearchDestinationApiVersion";
 export { ObservabilityPipelineElasticsearchDestinationAuth } from "./models/ObservabilityPipelineElasticsearchDestinationAuth";
+export { ObservabilityPipelineElasticsearchDestinationCompression } from "./models/ObservabilityPipelineElasticsearchDestinationCompression";
+export { ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm } from "./models/ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm";
 export { ObservabilityPipelineElasticsearchDestinationDataStream } from "./models/ObservabilityPipelineElasticsearchDestinationDataStream";
 export { ObservabilityPipelineElasticsearchDestinationType } from "./models/ObservabilityPipelineElasticsearchDestinationType";
 export { ObservabilityPipelineEnrichmentTableFile } from "./models/ObservabilityPipelineEnrichmentTableFile";

services/observability_pipelines/src/v2/models/ObservabilityPipelineConfigDestinationItem.ts

@@ -29,6 +29,7 @@ import { ObservabilityPipelineSyslogNgDestination } from "./ObservabilityPipelin
  * A destination for the pipeline.
  */
 export type ObservabilityPipelineConfigDestinationItem =
+  | ObservabilityPipelineElasticsearchDestination
   | ObservabilityPipelineHttpClientDestination
   | ObservabilityPipelineAmazonOpenSearchDestination
   | ObservabilityPipelineAmazonS3Destination
@@ -38,7 +39,6 @@ export type ObservabilityPipelineConfigDestinationItem =
   | ObservabilityPipelineCloudPremDestination
   | ObservabilityPipelineCrowdStrikeNextGenSiemDestination
   | ObservabilityPipelineDatadogLogsDestination
-  | ObservabilityPipelineElasticsearchDestination
   | ObservabilityPipelineGoogleChronicleDestination
   | ObservabilityPipelineGoogleCloudStorageDestination
   | ObservabilityPipelineGooglePubSubDestination

services/observability_pipelines/src/v2/models/ObservabilityPipelineElasticsearchDestination.ts

@@ -3,13 +3,15 @@ import { AttributeTypeMap } from "@datadog/datadog-api-client";
 import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
 import { ObservabilityPipelineElasticsearchDestinationApiVersion } from "./ObservabilityPipelineElasticsearchDestinationApiVersion";
 import { ObservabilityPipelineElasticsearchDestinationAuth } from "./ObservabilityPipelineElasticsearchDestinationAuth";
+import { ObservabilityPipelineElasticsearchDestinationCompression } from "./ObservabilityPipelineElasticsearchDestinationCompression";
 import { ObservabilityPipelineElasticsearchDestinationDataStream } from "./ObservabilityPipelineElasticsearchDestinationDataStream";
 import { ObservabilityPipelineElasticsearchDestinationType } from "./ObservabilityPipelineElasticsearchDestinationType";
+import { ObservabilityPipelineTls } from "./ObservabilityPipelineTls";
 
 /**
- * The `elasticsearch` destination writes logs to an Elasticsearch cluster.
+ * The `elasticsearch` destination writes logs or metrics to an Elasticsearch cluster.
  *
- * **Supported pipeline types:** logs
+ * **Supported pipeline types:** logs, metrics
  */
 export class ObservabilityPipelineElasticsearchDestination {
   /**
@@ -26,9 +28,13 @@ export class ObservabilityPipelineElasticsearchDestination {
    */
   "buffer"?: ObservabilityPipelineBufferOptions;
   /**
-   * The index to write logs to in Elasticsearch.
+   * The name of the index to write events to in Elasticsearch.
    */
   "bulkIndex"?: string;
+  /**
+   * Compression configuration for the Elasticsearch destination.
+   */
+  "compression"?: ObservabilityPipelineElasticsearchDestinationCompression;
   /**
    * Configuration options for writing to Elasticsearch Data Streams instead of a fixed index.
    */
@@ -41,10 +47,26 @@ export class ObservabilityPipelineElasticsearchDestination {
    * The unique identifier for this component.
    */
   "id": string;
+  /**
+   * The name of the field used as the document ID in Elasticsearch.
+   */
+  "idKey"?: string;
   /**
    * A list of component IDs whose output is used as the `input` for this component.
    */
   "inputs": Array<string>;
+  /**
+   * The name of an Elasticsearch ingest pipeline to apply to events before indexing.
+   */
+  "pipeline"?: string;
+  /**
+   * When `true`, retries failed partial bulk requests when some events in a batch fail while others succeed.
+   */
+  "requestRetryPartial"?: boolean;
+  /**
+   * Configuration for enabling TLS encryption between the pipeline component and external services.
+   */
+  "tls"?: ObservabilityPipelineTls;
   /**
    * The destination type. The value should always be `elasticsearch`.
    */
@@ -80,6 +102,10 @@ export class ObservabilityPipelineElasticsearchDestination {
       baseName: "bulk_index",
       type: "string",
     },
+    compression: {
+      baseName: "compression",
+      type: "ObservabilityPipelineElasticsearchDestinationCompression",
+    },
     dataStream: {
       baseName: "data_stream",
       type: "ObservabilityPipelineElasticsearchDestinationDataStream",
@@ -93,11 +119,27 @@ export class ObservabilityPipelineElasticsearchDestination {
       type: "string",
       required: true,
     },
+    idKey: {
+      baseName: "id_key",
+      type: "string",
+    },
     inputs: {
       baseName: "inputs",
       type: "Array<string>",
       required: true,
     },
+    pipeline: {
+      baseName: "pipeline",
+      type: "string",
+    },
+    requestRetryPartial: {
+      baseName: "request_retry_partial",
+      type: "boolean",
+    },
+    tls: {
+      baseName: "tls",
+      type: "ObservabilityPipelineTls",
+    },
     type: {
       baseName: "type",
       type: "ObservabilityPipelineElasticsearchDestinationType",

services/observability_pipelines/src/v2/models/ObservabilityPipelineElasticsearchDestinationCompression.ts

@@ -0,0 +1,56 @@
+import { AttributeTypeMap } from "@datadog/datadog-api-client";
+
+import { ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm } from "./ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm";
+
+/**
+ * Compression configuration for the Elasticsearch destination.
+ */
+export class ObservabilityPipelineElasticsearchDestinationCompression {
+  /**
+   * The compression algorithm applied when sending data to Elasticsearch.
+   */
+  "algorithm": ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm;
+  /**
+   * The compression level. Only applicable for `gzip`, `zlib`, and `zstd` algorithms.
+   */
+  "level"?: number;
+  /**
+   * A container for additional, undeclared properties.
+   * This is a holder for any undeclared properties as specified with
+   * the 'additionalProperties' keyword in the OAS document.
+   */
+  "additionalProperties"?: { [key: string]: any };
+  /**
+   * @ignore
+   */
+  "_unparsed"?: boolean;
+
+  /**
+   * @ignore
+   */
+  static readonly attributeTypeMap: AttributeTypeMap = {
+    algorithm: {
+      baseName: "algorithm",
+      type: "ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm",
+      required: true,
+    },
+    level: {
+      baseName: "level",
+      type: "number",
+      format: "int64",
+    },
+    additionalProperties: {
+      baseName: "additionalProperties",
+      type: "{ [key: string]: any; }",
+    },
+  };
+
+  /**
+   * @ignore
+   */
+  static getAttributeTypeMap(): AttributeTypeMap {
+    return ObservabilityPipelineElasticsearchDestinationCompression.attributeTypeMap;
+  }
+
+  public constructor() {}
+}

services/observability_pipelines/src/v2/models/ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm.ts

@@ -0,0 +1,17 @@
+import { UnparsedObject } from "@datadog/datadog-api-client";
+
+/**
+ * The compression algorithm applied when sending data to Elasticsearch.
+ */
+export type ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm =
+  | typeof NONE
+  | typeof GZIP
+  | typeof ZLIB
+  | typeof ZSTD
+  | typeof SNAPPY
+  | UnparsedObject;
+export const NONE = "none";
+export const GZIP = "gzip";
+export const ZLIB = "zlib";
+export const ZSTD = "zstd";
+export const SNAPPY = "snappy";

services/observability_pipelines/src/v2/models/ObservabilityPipelineElasticsearchDestinationDataStream.ts

@@ -5,17 +5,25 @@ import { AttributeTypeMap } from "@datadog/datadog-api-client";
  */
 export class ObservabilityPipelineElasticsearchDestinationDataStream {
   /**
-   * The data stream dataset for your logs. This groups logs by their source or application.
+   * When `true`, automatically routes events to the appropriate data stream based on the event content.
+   */
+  "autoRouting"?: boolean;
+  /**
+   * The data stream dataset. This groups events by their source or application.
    */
   "dataset"?: string;
   /**
-   * The data stream type for your logs. This determines how logs are categorized within the data stream.
+   * The data stream type. This determines how events are categorized within the data stream.
    */
   "dtype"?: string;
   /**
-   * The data stream namespace for your logs. This separates logs into different environments or domains.
+   * The data stream namespace. This separates events into different environments or domains.
    */
   "namespace"?: string;
+  /**
+   * When `true`, synchronizes data stream fields with the Elasticsearch index mapping.
+   */
+  "syncFields"?: boolean;
   /**
    * A container for additional, undeclared properties.
    * This is a holder for any undeclared properties as specified with
@@ -31,6 +39,10 @@ export class ObservabilityPipelineElasticsearchDestinationDataStream {
    * @ignore
    */
   static readonly attributeTypeMap: AttributeTypeMap = {
+    autoRouting: {
+      baseName: "auto_routing",
+      type: "boolean",
+    },
     dataset: {
       baseName: "dataset",
       type: "string",
@@ -43,6 +55,10 @@ export class ObservabilityPipelineElasticsearchDestinationDataStream {
       baseName: "namespace",
       type: "string",
     },
+    syncFields: {
+      baseName: "sync_fields",
+      type: "boolean",
+    },
     additionalProperties: {
       baseName: "additionalProperties",
       type: "{ [key: string]: any; }",