Add Obs Pipelines Elasticsearch metrics destination and new fields (DataDog#3922)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -43722,6 +43722,7 @@ components:
     ObservabilityPipelineConfigDestinationItem:
       description: "A destination for the pipeline."
       oneOf:
+        - $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineAmazonOpenSearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineAmazonS3Destination"
@@ -43731,7 +43732,6 @@ components:
         - $ref: "#/components/schemas/ObservabilityPipelineCloudPremDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineDatadogLogsDestination"
-        - $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGoogleChronicleDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGoogleCloudStorageDestination"
         - $ref: "#/components/schemas/ObservabilityPipelineGooglePubSubDestination"
@@ -44333,9 +44333,9 @@ components:
       type: object
     ObservabilityPipelineElasticsearchDestination:
       description: |-
-        The `elasticsearch` destination writes logs to an Elasticsearch cluster.
+        The `elasticsearch` destination writes logs or metrics to an Elasticsearch cluster.
 
-        **Supported pipeline types:** logs
+        **Supported pipeline types:** logs, metrics
       properties:
         api_version:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationApiVersion"
@@ -44344,9 +44344,11 @@ components:
         buffer:
           $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         bulk_index:
-          description: The index to write logs to in Elasticsearch.
+          description: The name of the index to write events to in Elasticsearch.
           example: logs-index
           type: string
+        compression:
+          $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationCompression"
         data_stream:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationDataStream"
         endpoint_url_key:
@@ -44357,21 +44359,34 @@ components:
           description: The unique identifier for this component.
           example: "elasticsearch-destination"
           type: string
+        id_key:
+          description: The name of the field used as the document ID in Elasticsearch.
+          example: id
+          type: string
         inputs:
           description: A list of component IDs whose output is used as the `input` for this component.
           example: ["filter-processor"]
           items:
             description: The ID of a component whose output is used as input for this destination.
             type: string
           type: array
+        pipeline:
+          description: The name of an Elasticsearch ingest pipeline to apply to events before indexing.
+          example: my-pipeline
+          type: string
+        request_retry_partial:
+          description: When `true`, retries failed partial bulk requests when some events in a batch fail while others succeed.
+          type: boolean
+        tls:
+          $ref: "#/components/schemas/ObservabilityPipelineTls"
         type:
           $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationType"
       required:
         - id
         - type
         - inputs
       type: object
-      x-pipeline-types: [logs]
+      x-pipeline-types: [logs, metrics]
     ObservabilityPipelineElasticsearchDestinationApiVersion:
       description: The Elasticsearch API version to use. Set to `auto` to auto-detect.
       enum: [auto, v6, v7, v8]
@@ -44400,18 +44415,48 @@ components:
       required:
         - strategy
       type: object
+    ObservabilityPipelineElasticsearchDestinationCompression:
+      description: Compression configuration for the Elasticsearch destination.
+      properties:
+        algorithm:
+          $ref: "#/components/schemas/ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm"
+        level:
+          description: The compression level. Only applicable for `gzip`, `zlib`, and `zstd` algorithms.
+          example: 6
+          format: int64
+          type: integer
+      required:
+        - algorithm
+      type: object
+    ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm:
+      description: The compression algorithm applied when sending data to Elasticsearch.
+      enum: [none, gzip, zlib, zstd, snappy]
+      example: gzip
+      type: string
+      x-enum-varnames:
+        - NONE
+        - GZIP
+        - ZLIB
+        - ZSTD
+        - SNAPPY
     ObservabilityPipelineElasticsearchDestinationDataStream:
       description: Configuration options for writing to Elasticsearch Data Streams instead of a fixed index.
       properties:
+        auto_routing:
+          description: When `true`, automatically routes events to the appropriate data stream based on the event content.
+          type: boolean
         dataset:
-          description: The data stream dataset for your logs. This groups logs by their source or application.
+          description: The data stream dataset. This groups events by their source or application.
           type: string
         dtype:
-          description: The data stream type for your logs. This determines how logs are categorized within the data stream.
+          description: The data stream type. This determines how events are categorized within the data stream.
           type: string
         namespace:
-          description: The data stream namespace for your logs. This separates logs into different environments or domains.
+          description: The data stream namespace. This separates events into different environments or domains.
           type: string
+        sync_fields:
+          description: When `true`, synchronizes data stream fields with the Elasticsearch index mapping.
+          type: boolean
       type: object
     ObservabilityPipelineElasticsearchDestinationType:
       default: elasticsearch

api/datadogV2/model_observability_pipeline_config_destination_item.go

@@ -10,6 +10,7 @@ import (
 
 // ObservabilityPipelineConfigDestinationItem - A destination for the pipeline.
 type ObservabilityPipelineConfigDestinationItem struct {
+	ObservabilityPipelineElasticsearchDestination          *ObservabilityPipelineElasticsearchDestination
 	ObservabilityPipelineHttpClientDestination             *ObservabilityPipelineHttpClientDestination
 	ObservabilityPipelineAmazonOpenSearchDestination       *ObservabilityPipelineAmazonOpenSearchDestination
 	ObservabilityPipelineAmazonS3Destination               *ObservabilityPipelineAmazonS3Destination
@@ -19,7 +20,6 @@ type ObservabilityPipelineConfigDestinationItem struct {
 	ObservabilityPipelineCloudPremDestination              *ObservabilityPipelineCloudPremDestination
 	ObservabilityPipelineCrowdStrikeNextGenSiemDestination *ObservabilityPipelineCrowdStrikeNextGenSiemDestination
 	ObservabilityPipelineDatadogLogsDestination            *ObservabilityPipelineDatadogLogsDestination
-	ObservabilityPipelineElasticsearchDestination          *ObservabilityPipelineElasticsearchDestination
 	ObservabilityPipelineGoogleChronicleDestination        *ObservabilityPipelineGoogleChronicleDestination
 	ObservabilityPipelineGoogleCloudStorageDestination     *ObservabilityPipelineGoogleCloudStorageDestination
 	ObservabilityPipelineGooglePubSubDestination           *ObservabilityPipelineGooglePubSubDestination
@@ -39,6 +39,11 @@ type ObservabilityPipelineConfigDestinationItem struct {
 	UnparsedObject interface{}
 }
 
+// ObservabilityPipelineElasticsearchDestinationAsObservabilityPipelineConfigDestinationItem is a convenience function that returns ObservabilityPipelineElasticsearchDestination wrapped in ObservabilityPipelineConfigDestinationItem.
+func ObservabilityPipelineElasticsearchDestinationAsObservabilityPipelineConfigDestinationItem(v *ObservabilityPipelineElasticsearchDestination) ObservabilityPipelineConfigDestinationItem {
+	return ObservabilityPipelineConfigDestinationItem{ObservabilityPipelineElasticsearchDestination: v}
+}
+
 // ObservabilityPipelineHttpClientDestinationAsObservabilityPipelineConfigDestinationItem is a convenience function that returns ObservabilityPipelineHttpClientDestination wrapped in ObservabilityPipelineConfigDestinationItem.
 func ObservabilityPipelineHttpClientDestinationAsObservabilityPipelineConfigDestinationItem(v *ObservabilityPipelineHttpClientDestination) ObservabilityPipelineConfigDestinationItem {
 	return ObservabilityPipelineConfigDestinationItem{ObservabilityPipelineHttpClientDestination: v}
@@ -84,11 +89,6 @@ func ObservabilityPipelineDatadogLogsDestinationAsObservabilityPipelineConfigDes
 	return ObservabilityPipelineConfigDestinationItem{ObservabilityPipelineDatadogLogsDestination: v}
 }
 
-// ObservabilityPipelineElasticsearchDestinationAsObservabilityPipelineConfigDestinationItem is a convenience function that returns ObservabilityPipelineElasticsearchDestination wrapped in ObservabilityPipelineConfigDestinationItem.
-func ObservabilityPipelineElasticsearchDestinationAsObservabilityPipelineConfigDestinationItem(v *ObservabilityPipelineElasticsearchDestination) ObservabilityPipelineConfigDestinationItem {
-	return ObservabilityPipelineConfigDestinationItem{ObservabilityPipelineElasticsearchDestination: v}
-}
-
 // ObservabilityPipelineGoogleChronicleDestinationAsObservabilityPipelineConfigDestinationItem is a convenience function that returns ObservabilityPipelineGoogleChronicleDestination wrapped in ObservabilityPipelineConfigDestinationItem.
 func ObservabilityPipelineGoogleChronicleDestinationAsObservabilityPipelineConfigDestinationItem(v *ObservabilityPipelineGoogleChronicleDestination) ObservabilityPipelineConfigDestinationItem {
 	return ObservabilityPipelineConfigDestinationItem{ObservabilityPipelineGoogleChronicleDestination: v}
@@ -163,6 +163,23 @@ func ObservabilityPipelineDatadogMetricsDestinationAsObservabilityPipelineConfig
 func (obj *ObservabilityPipelineConfigDestinationItem) UnmarshalJSON(data []byte) error {
 	var err error
 	match := 0
+	// try to unmarshal data into ObservabilityPipelineElasticsearchDestination
+	err = datadog.Unmarshal(data, &obj.ObservabilityPipelineElasticsearchDestination)
+	if err == nil {
+		if obj.ObservabilityPipelineElasticsearchDestination != nil && obj.ObservabilityPipelineElasticsearchDestination.UnparsedObject == nil {
+			jsonObservabilityPipelineElasticsearchDestination, _ := datadog.Marshal(obj.ObservabilityPipelineElasticsearchDestination)
+			if string(jsonObservabilityPipelineElasticsearchDestination) == "{}" { // empty struct
+				obj.ObservabilityPipelineElasticsearchDestination = nil
+			} else {
+				match++
+			}
+		} else {
+			obj.ObservabilityPipelineElasticsearchDestination = nil
+		}
+	} else {
+		obj.ObservabilityPipelineElasticsearchDestination = nil
+	}
+
 	// try to unmarshal data into ObservabilityPipelineHttpClientDestination
 	err = datadog.Unmarshal(data, &obj.ObservabilityPipelineHttpClientDestination)
 	if err == nil {
@@ -316,23 +333,6 @@ func (obj *ObservabilityPipelineConfigDestinationItem) UnmarshalJSON(data []byte
 		obj.ObservabilityPipelineDatadogLogsDestination = nil
 	}
 
-	// try to unmarshal data into ObservabilityPipelineElasticsearchDestination
-	err = datadog.Unmarshal(data, &obj.ObservabilityPipelineElasticsearchDestination)
-	if err == nil {
-		if obj.ObservabilityPipelineElasticsearchDestination != nil && obj.ObservabilityPipelineElasticsearchDestination.UnparsedObject == nil {
-			jsonObservabilityPipelineElasticsearchDestination, _ := datadog.Marshal(obj.ObservabilityPipelineElasticsearchDestination)
-			if string(jsonObservabilityPipelineElasticsearchDestination) == "{}" { // empty struct
-				obj.ObservabilityPipelineElasticsearchDestination = nil
-			} else {
-				match++
-			}
-		} else {
-			obj.ObservabilityPipelineElasticsearchDestination = nil
-		}
-	} else {
-		obj.ObservabilityPipelineElasticsearchDestination = nil
-	}
-
 	// try to unmarshal data into ObservabilityPipelineGoogleChronicleDestination
 	err = datadog.Unmarshal(data, &obj.ObservabilityPipelineGoogleChronicleDestination)
 	if err == nil {
@@ -573,6 +573,7 @@ func (obj *ObservabilityPipelineConfigDestinationItem) UnmarshalJSON(data []byte
 
 	if match != 1 { // more than 1 match
 		// reset to nil
+		obj.ObservabilityPipelineElasticsearchDestination = nil
 		obj.ObservabilityPipelineHttpClientDestination = nil
 		obj.ObservabilityPipelineAmazonOpenSearchDestination = nil
 		obj.ObservabilityPipelineAmazonS3Destination = nil
@@ -582,7 +583,6 @@ func (obj *ObservabilityPipelineConfigDestinationItem) UnmarshalJSON(data []byte
 		obj.ObservabilityPipelineCloudPremDestination = nil
 		obj.ObservabilityPipelineCrowdStrikeNextGenSiemDestination = nil
 		obj.ObservabilityPipelineDatadogLogsDestination = nil
-		obj.ObservabilityPipelineElasticsearchDestination = nil
 		obj.ObservabilityPipelineGoogleChronicleDestination = nil
 		obj.ObservabilityPipelineGoogleCloudStorageDestination = nil
 		obj.ObservabilityPipelineGooglePubSubDestination = nil
@@ -604,6 +604,10 @@ func (obj *ObservabilityPipelineConfigDestinationItem) UnmarshalJSON(data []byte
 
 // MarshalJSON turns data from the first non-nil pointers in the struct to JSON.
 func (obj ObservabilityPipelineConfigDestinationItem) MarshalJSON() ([]byte, error) {
+	if obj.ObservabilityPipelineElasticsearchDestination != nil {
+		return datadog.Marshal(&obj.ObservabilityPipelineElasticsearchDestination)
+	}
+
 	if obj.ObservabilityPipelineHttpClientDestination != nil {
 		return datadog.Marshal(&obj.ObservabilityPipelineHttpClientDestination)
 	}
@@ -640,10 +644,6 @@ func (obj ObservabilityPipelineConfigDestinationItem) MarshalJSON() ([]byte, err
 		return datadog.Marshal(&obj.ObservabilityPipelineDatadogLogsDestination)
 	}
 
-	if obj.ObservabilityPipelineElasticsearchDestination != nil {
-		return datadog.Marshal(&obj.ObservabilityPipelineElasticsearchDestination)
-	}
-
 	if obj.ObservabilityPipelineGoogleChronicleDestination != nil {
 		return datadog.Marshal(&obj.ObservabilityPipelineGoogleChronicleDestination)
 	}
@@ -708,6 +708,10 @@ func (obj ObservabilityPipelineConfigDestinationItem) MarshalJSON() ([]byte, err
 
 // GetActualInstance returns the actual instance.
 func (obj *ObservabilityPipelineConfigDestinationItem) GetActualInstance() interface{} {
+	if obj.ObservabilityPipelineElasticsearchDestination != nil {
+		return obj.ObservabilityPipelineElasticsearchDestination
+	}
+
 	if obj.ObservabilityPipelineHttpClientDestination != nil {
 		return obj.ObservabilityPipelineHttpClientDestination
 	}
@@ -744,10 +748,6 @@ func (obj *ObservabilityPipelineConfigDestinationItem) GetActualInstance() inter
 		return obj.ObservabilityPipelineDatadogLogsDestination
 	}
 
-	if obj.ObservabilityPipelineElasticsearchDestination != nil {
-		return obj.ObservabilityPipelineElasticsearchDestination
-	}
-
 	if obj.ObservabilityPipelineGoogleChronicleDestination != nil {
 		return obj.ObservabilityPipelineGoogleChronicleDestination
 	}