Adding Buffer options to Observability Pipeline Destinations (#3427)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -6766,6 +6766,8 @@ components:
           description: Optional prefix for blobs written to the container.
           example: logs/
           type: string
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         container_name:
           description: The name of the Azure Blob Storage container to store logs
             in.
@@ -36391,6 +36393,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         client_id:
           description: Azure AD client ID used for authentication.
           example: a1b2c3d4-5678-90ab-cdef-1234567890ab
@@ -38337,6 +38341,8 @@ components:
       properties:
         auth:
           $ref: '#/components/schemas/ObservabilityPipelineAmazonOpenSearchDestinationAuth'
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         bulk_index:
           description: The index to write logs to.
           example: logs-index
@@ -38418,6 +38424,8 @@ components:
           description: S3 bucket name.
           example: error-logs
           type: string
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: Unique identifier for the destination component.
           example: amazon-s3-destination
@@ -38539,6 +38547,8 @@ components:
           description: Name of the Amazon S3 bucket in Security Lake (3-63 characters).
           example: security-lake-bucket
           type: string
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         custom_source_name:
           description: Custom source name for the logs in Security Lake.
           example: my-custom-source
@@ -38598,6 +38608,39 @@ components:
             role session.
           type: string
       type: object
+    ObservabilityPipelineBufferOptions:
+      description: Configuration for buffer settings on destination components.
+      oneOf:
+      - $ref: '#/components/schemas/ObservabilityPipelineDiskBufferOptions'
+      - $ref: '#/components/schemas/ObservabilityPipelineMemoryBufferOptions'
+      - $ref: '#/components/schemas/ObservabilityPipelineMemoryBufferSizeOptions'
+    ObservabilityPipelineBufferOptionsDiskType:
+      default: disk
+      description: The type of the buffer that will be configured, a disk buffer.
+      enum:
+      - disk
+      type: string
+      x-enum-varnames:
+      - DISK
+    ObservabilityPipelineBufferOptionsMemoryType:
+      default: memory
+      description: The type of the buffer that will be configured, a memory buffer.
+      enum:
+      - memory
+      type: string
+      x-enum-varnames:
+      - MEMORY
+    ObservabilityPipelineBufferOptionsWhenFull:
+      default: block
+      description: Behavior when the buffer is full (block and stop accepting new
+        events, or drop new events)
+      enum:
+      - block
+      - drop_newest
+      type: string
+      x-enum-varnames:
+      - BLOCK
+      - DROP_NEWEST
     ObservabilityPipelineCloudPremDestination:
       description: 'The `cloud_prem` destination sends logs to Datadog CloudPrem.
 
@@ -38868,6 +38911,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         compression:
           $ref: '#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestinationCompression'
         encoding:
@@ -39093,6 +39138,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: The unique identifier for this component.
           example: datadog-logs-destination
@@ -39347,6 +39394,19 @@ components:
       type: string
       x-enum-varnames:
       - DEDUPE
+    ObservabilityPipelineDiskBufferOptions:
+      description: Options for configuring a disk buffer.
+      properties:
+        max_size:
+          description: Maximum size of the disk buffer.
+          example: 4096
+          format: int64
+          type: integer
+        type:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptionsDiskType'
+        when_full:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptionsWhenFull'
+      type: object
     ObservabilityPipelineElasticsearchDestination:
       description: 'The `elasticsearch` destination writes logs to an Elasticsearch
         cluster.
@@ -39356,6 +39416,8 @@ components:
       properties:
         api_version:
           $ref: '#/components/schemas/ObservabilityPipelineElasticsearchDestinationApiVersion'
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         bulk_index:
           description: The index to write logs to in Elasticsearch.
           example: logs-index
@@ -39901,6 +39963,8 @@ components:
       properties:
         auth:
           $ref: '#/components/schemas/ObservabilityPipelineGcpAuth'
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         customer_id:
           description: The Google Chronicle customer ID.
           example: abcdefg123456789
@@ -39969,6 +40033,8 @@ components:
           description: Name of the GCS bucket.
           example: error-logs
           type: string
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: Unique identifier for the destination component.
           example: gcs-destination
@@ -40053,6 +40119,8 @@ components:
       properties:
         auth:
           $ref: '#/components/schemas/ObservabilityPipelineGcpAuth'
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         encoding:
           $ref: '#/components/schemas/ObservabilityPipelineGooglePubSubDestinationEncoding'
         id:
@@ -40579,6 +40647,28 @@ components:
       type: string
       x-enum-varnames:
       - LOGSTASH
+    ObservabilityPipelineMemoryBufferOptions:
+      description: Options for configuring a memory buffer by byte size.
+      properties:
+        max_size:
+          description: Maximum size of the memory buffer.
+          example: 4096
+          format: int64
+          type: integer
+        type:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
+      type: object
+    ObservabilityPipelineMemoryBufferSizeOptions:
+      description: Options for configuring a memory buffer by queue length.
+      properties:
+        max_events:
+          description: Maximum events for the memory buffer.
+          example: 500
+          format: int64
+          type: integer
+        type:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptionsMemoryType'
+      type: object
     ObservabilityPipelineMetadataEntry:
       description: A custom metadata entry.
       properties:
@@ -40701,6 +40791,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: The unique identifier for this component.
           example: new-relic-destination
@@ -40847,6 +40939,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         bulk_index:
           description: The index to write logs to.
           example: logs-index
@@ -41560,6 +41654,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: The unique identifier for this component.
           example: rsyslog-destination
@@ -42065,6 +42161,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: The unique identifier for this component.
           example: sentinelone-destination
@@ -42119,6 +42217,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         encoding:
           $ref: '#/components/schemas/ObservabilityPipelineSocketDestinationEncoding'
         framing:
@@ -42495,6 +42595,8 @@ components:
             If `false`, Splunk assigns the time the event was received.'
           example: true
           type: boolean
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         encoding:
           $ref: '#/components/schemas/ObservabilityPipelineSplunkHecDestinationEncoding'
         id:
@@ -42619,6 +42721,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         encoding:
           $ref: '#/components/schemas/ObservabilityPipelineSumoLogicDestinationEncoding'
         header_custom_fields:
@@ -42732,6 +42836,8 @@ components:
 
         **Supported pipeline types:** logs'
       properties:
+        buffer:
+          $ref: '#/components/schemas/ObservabilityPipelineBufferOptions'
         id:
           description: The unique identifier for this component.
           example: syslog-ng-destination

services/observability_pipelines/src/v2/index.ts

@@ -37,6 +37,10 @@ export { ObservabilityPipelineAmazonS3SourceType } from "./models/ObservabilityP
 export { ObservabilityPipelineAmazonSecurityLakeDestination } from "./models/ObservabilityPipelineAmazonSecurityLakeDestination";
 export { ObservabilityPipelineAmazonSecurityLakeDestinationType } from "./models/ObservabilityPipelineAmazonSecurityLakeDestinationType";
 export { ObservabilityPipelineAwsAuth } from "./models/ObservabilityPipelineAwsAuth";
+export { ObservabilityPipelineBufferOptions } from "./models/ObservabilityPipelineBufferOptions";
+export { ObservabilityPipelineBufferOptionsDiskType } from "./models/ObservabilityPipelineBufferOptionsDiskType";
+export { ObservabilityPipelineBufferOptionsMemoryType } from "./models/ObservabilityPipelineBufferOptionsMemoryType";
+export { ObservabilityPipelineBufferOptionsWhenFull } from "./models/ObservabilityPipelineBufferOptionsWhenFull";
 export { ObservabilityPipelineCloudPremDestination } from "./models/ObservabilityPipelineCloudPremDestination";
 export { ObservabilityPipelineCloudPremDestinationType } from "./models/ObservabilityPipelineCloudPremDestinationType";
 export { ObservabilityPipelineConfig } from "./models/ObservabilityPipelineConfig";
@@ -70,6 +74,7 @@ export { ObservabilityPipelineDecoding } from "./models/ObservabilityPipelineDec
 export { ObservabilityPipelineDedupeProcessor } from "./models/ObservabilityPipelineDedupeProcessor";
 export { ObservabilityPipelineDedupeProcessorMode } from "./models/ObservabilityPipelineDedupeProcessorMode";
 export { ObservabilityPipelineDedupeProcessorType } from "./models/ObservabilityPipelineDedupeProcessorType";
+export { ObservabilityPipelineDiskBufferOptions } from "./models/ObservabilityPipelineDiskBufferOptions";
 export { ObservabilityPipelineElasticsearchDestination } from "./models/ObservabilityPipelineElasticsearchDestination";
 export { ObservabilityPipelineElasticsearchDestinationApiVersion } from "./models/ObservabilityPipelineElasticsearchDestinationApiVersion";
 export { ObservabilityPipelineElasticsearchDestinationDataStream } from "./models/ObservabilityPipelineElasticsearchDestinationDataStream";
@@ -136,6 +141,8 @@ export { ObservabilityPipelineKafkaSource } from "./models/ObservabilityPipeline
 export { ObservabilityPipelineKafkaSourceType } from "./models/ObservabilityPipelineKafkaSourceType";
 export { ObservabilityPipelineLogstashSource } from "./models/ObservabilityPipelineLogstashSource";
 export { ObservabilityPipelineLogstashSourceType } from "./models/ObservabilityPipelineLogstashSourceType";
+export { ObservabilityPipelineMemoryBufferOptions } from "./models/ObservabilityPipelineMemoryBufferOptions";
+export { ObservabilityPipelineMemoryBufferSizeOptions } from "./models/ObservabilityPipelineMemoryBufferSizeOptions";
 export { ObservabilityPipelineMetadataEntry } from "./models/ObservabilityPipelineMetadataEntry";
 export { ObservabilityPipelineMetricTagsProcessor } from "./models/ObservabilityPipelineMetricTagsProcessor";
 export { ObservabilityPipelineMetricTagsProcessorRule } from "./models/ObservabilityPipelineMetricTagsProcessorRule";

services/observability_pipelines/src/v2/models/AzureStorageDestination.ts

@@ -1,6 +1,7 @@
 import { AttributeTypeMap } from "@datadog/datadog-api-client";
 
 import { AzureStorageDestinationType } from "./AzureStorageDestinationType";
+import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
 
 /**
  * The `azure_storage` destination forwards logs to an Azure Blob Storage container.
@@ -12,6 +13,10 @@ export class AzureStorageDestination {
    * Optional prefix for blobs written to the container.
    */
   "blobPrefix"?: string;
+  /**
+   * Configuration for buffer settings on destination components.
+   */
+  "buffer"?: ObservabilityPipelineBufferOptions;
   /**
    * The name of the Azure Blob Storage container to store logs in.
    */
@@ -47,6 +52,10 @@ export class AzureStorageDestination {
       baseName: "blob_prefix",
       type: "string",
     },
+    buffer: {
+      baseName: "buffer",
+      type: "ObservabilityPipelineBufferOptions",
+    },
     containerName: {
       baseName: "container_name",
       type: "string",

services/observability_pipelines/src/v2/models/MicrosoftSentinelDestination.ts

@@ -1,13 +1,18 @@
 import { AttributeTypeMap } from "@datadog/datadog-api-client";
 
 import { MicrosoftSentinelDestinationType } from "./MicrosoftSentinelDestinationType";
+import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
 
 /**
  * The `microsoft_sentinel` destination forwards logs to Microsoft Sentinel.
  *
  * **Supported pipeline types:** logs
  */
 export class MicrosoftSentinelDestination {
+  /**
+   * Configuration for buffer settings on destination components.
+   */
+  "buffer"?: ObservabilityPipelineBufferOptions;
   /**
    * Azure AD client ID used for authentication.
    */
@@ -51,6 +56,10 @@ export class MicrosoftSentinelDestination {
    * @ignore
    */
   static readonly attributeTypeMap: AttributeTypeMap = {
+    buffer: {
+      baseName: "buffer",
+      type: "ObservabilityPipelineBufferOptions",
+    },
     clientId: {
       baseName: "client_id",
       type: "string",

services/observability_pipelines/src/v2/models/ObservabilityPipelineAmazonOpenSearchDestination.ts

@@ -2,6 +2,7 @@ import { AttributeTypeMap } from "@datadog/datadog-api-client";
 
 import { ObservabilityPipelineAmazonOpenSearchDestinationAuth } from "./ObservabilityPipelineAmazonOpenSearchDestinationAuth";
 import { ObservabilityPipelineAmazonOpenSearchDestinationType } from "./ObservabilityPipelineAmazonOpenSearchDestinationType";
+import { ObservabilityPipelineBufferOptions } from "./ObservabilityPipelineBufferOptions";
 
 /**
  * The `amazon_opensearch` destination writes logs to Amazon OpenSearch.
@@ -14,6 +15,10 @@ export class ObservabilityPipelineAmazonOpenSearchDestination {
    * The `strategy` field determines whether basic or AWS-based authentication is used.
    */
   "auth": ObservabilityPipelineAmazonOpenSearchDestinationAuth;
+  /**
+   * Configuration for buffer settings on destination components.
+   */
+  "buffer"?: ObservabilityPipelineBufferOptions;
   /**
    * The index to write logs to.
    */
@@ -50,6 +55,10 @@ export class ObservabilityPipelineAmazonOpenSearchDestination {
       type: "ObservabilityPipelineAmazonOpenSearchDestinationAuth",
       required: true,
     },
+    buffer: {
+      baseName: "buffer",
+      type: "ObservabilityPipelineBufferOptions",
+    },
     bulkIndex: {
       baseName: "bulk_index",
       type: "string",