Skip to content

Commit 9f86099

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for cache property in dedupe processor in observability pipelines (#3451)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent e6aa5f9 commit 9f86099

10 files changed

Lines changed: 312 additions & 0 deletions

File tree

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39514,6 +39514,8 @@ components:
3951439514

3951539515
**Supported pipeline types:** logs'
3951639516
properties:
39517+
cache:
39518+
$ref: '#/components/schemas/ObservabilityPipelineDedupeProcessorCache'
3951739519
display_name:
3951839520
$ref: '#/components/schemas/ObservabilityPipelineComponentDisplayName'
3951939521
enabled:
@@ -39551,6 +39553,19 @@ components:
3955139553
type: object
3955239554
x-pipeline-types:
3955339555
- logs
39556+
ObservabilityPipelineDedupeProcessorCache:
39557+
description: Configuration for the cache used to detect duplicates.
39558+
properties:
39559+
num_events:
39560+
description: The number of events to cache for duplicate detection.
39561+
example: 5000
39562+
format: int64
39563+
maximum: 1000000000
39564+
minimum: 1
39565+
type: integer
39566+
required:
39567+
- num_events
39568+
type: object
3955439569
ObservabilityPipelineDedupeProcessorMode:
3955539570
description: The deduplication mode to apply to the fields.
3955639571
enum:

‎cassettes/v2/Observability-Pipelines_4170000189/Create-a-pipeline-with-dedupe-processor-with-cache-returns-OK-response_3441761873/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-02-09T09:53:31.840Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Create-a-pipeline-with-dedupe-processor-with-cache-returns-OK-response_3441761873/recording.har‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Create a pipeline with dedupe processor with cache returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "5e54c4e68df050e5e062b51538d22d19",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 557,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 572,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"cache\":{\"num_events\":5000},\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with Dedupe Cache\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines"
39+
},
40+
"response": {
41+
"bodySize": 892,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 892,
45+
"text": "{\"data\":{\"id\":\"31a750dc-059d-11f1-a2a8-da7ad0900002\",\"type\":\"pipelines\",\"attributes\":{\"name\":\"Pipeline with Dedupe Cache\",\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"cache\":{\"num_events\":5000},\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"processors\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"cache\":{\"num_events\":5000},\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]}}}}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 371,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 201,
58+
"statusText": "Created"
59+
},
60+
"startedDateTime": "2026-02-09T09:53:32.500Z",
61+
"time": 804
62+
},
63+
{
64+
"_id": "5d43f0252ced2d8416615eb0c54f44bb",
65+
"_order": 0,
66+
"cache": {},
67+
"request": {
68+
"bodySize": 0,
69+
"cookies": [],
70+
"headers": [
71+
{
72+
"_fromType": "array",
73+
"name": "accept",
74+
"value": "*/*"
75+
}
76+
],
77+
"headersSize": 545,
78+
"httpVersion": "HTTP/1.1",
79+
"method": "DELETE",
80+
"queryString": [],
81+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/31a750dc-059d-11f1-a2a8-da7ad0900002"
82+
},
83+
"response": {
84+
"bodySize": 0,
85+
"content": {
86+
"mimeType": "application/json",
87+
"size": 0
88+
},
89+
"cookies": [],
90+
"headers": [
91+
{
92+
"name": "content-type",
93+
"value": "application/json"
94+
}
95+
],
96+
"headersSize": 342,
97+
"httpVersion": "HTTP/1.1",
98+
"redirectURL": "",
99+
"status": 204,
100+
"statusText": "No Content"
101+
},
102+
"startedDateTime": "2026-02-09T09:53:33.313Z",
103+
"time": 625
104+
}
105+
],
106+
"pages": [],
107+
"version": "1.2"
108+
}
109+
}

‎cassettes/v2/Observability-Pipelines_4170000189/Create-a-pipeline-with-dedupe-processor-without-cache-returns-OK-response_76456799/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-02-09T09:53:33.945Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Create-a-pipeline-with-dedupe-processor-without-cache-returns-OK-response_76456799/recording.har‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Create a pipeline with dedupe processor without cache returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "49b5783e758548037f434f9c3b65f4ef",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 532,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 572,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with Dedupe No Cache\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines"
39+
},
40+
"response": {
41+
"bodySize": 839,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 839,
45+
"text": "{\"data\":{\"id\":\"3280ccb8-059d-11f1-a2aa-da7ad0900002\",\"type\":\"pipelines\",\"attributes\":{\"name\":\"Pipeline with Dedupe No Cache\",\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"processors\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"fields\":[\"message\"],\"id\":\"dedupe-processor\",\"include\":\"service:my-service\",\"mode\":\"match\",\"type\":\"dedupe\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]}}}}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 371,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 201,
58+
"statusText": "Created"
59+
},
60+
"startedDateTime": "2026-02-09T09:53:33.947Z",
61+
"time": 606
62+
},
63+
{
64+
"_id": "ecf742e66517f10dd4c857002c94f59a",
65+
"_order": 0,
66+
"cache": {},
67+
"request": {
68+
"bodySize": 0,
69+
"cookies": [],
70+
"headers": [
71+
{
72+
"_fromType": "array",
73+
"name": "accept",
74+
"value": "*/*"
75+
}
76+
],
77+
"headersSize": 545,
78+
"httpVersion": "HTTP/1.1",
79+
"method": "DELETE",
80+
"queryString": [],
81+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/3280ccb8-059d-11f1-a2aa-da7ad0900002"
82+
},
83+
"response": {
84+
"bodySize": 0,
85+
"content": {
86+
"mimeType": "application/json",
87+
"size": 0
88+
},
89+
"cookies": [],
90+
"headers": [
91+
{
92+
"name": "content-type",
93+
"value": "application/json"
94+
}
95+
],
96+
"headersSize": 342,
97+
"httpVersion": "HTTP/1.1",
98+
"redirectURL": "",
99+
"status": 204,
100+
"statusText": "No Content"
101+
},
102+
"startedDateTime": "2026-02-09T09:53:34.558Z",
103+
"time": 504
104+
}
105+
],
106+
"pages": [],
107+
"version": "1.2"
108+
}
109+
}

‎features/v2/observability_pipelines.feature‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,24 @@ Feature: Observability Pipelines
3535
And the response "data.attributes.config.processor_groups" has length 1
3636
And the response "data.attributes.config.destinations" has length 1
3737

38+
@team:DataDog/observability-pipelines
39+
Scenario: Create a pipeline with dedupe processor with cache returns "OK" response
40+
Given new "CreatePipeline" request
41+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "dedupe-processor", "include": "service:my-service", "type": "dedupe", "fields": ["message"], "mode": "match", "cache": {"num_events": 5000}}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Dedupe Cache"}, "type": "pipelines"}}
42+
When the request is sent
43+
Then the response status is 201 OK
44+
And the response "data.attributes.config.processor_groups[0].processors[0].type" is equal to "dedupe"
45+
And the response "data.attributes.config.processor_groups[0].processors[0].cache.num_events" is equal to 5000
46+
47+
@team:DataDog/observability-pipelines
48+
Scenario: Create a pipeline with dedupe processor without cache returns "OK" response
49+
Given new "CreatePipeline" request
50+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "dedupe-processor", "include": "service:my-service", "type": "dedupe", "fields": ["message"], "mode": "match"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Dedupe No Cache"}, "type": "pipelines"}}
51+
When the request is sent
52+
Then the response status is 201 OK
53+
And the response "data.attributes.config.processor_groups[0].processors[0].type" is equal to "dedupe"
54+
And the response "data.attributes.config.processor_groups[0].processors[0].fields[0]" is equal to "message"
55+
3856
@generated @skip @team:DataDog/observability-pipelines
3957
Scenario: Delete a pipeline returns "Conflict" response
4058
Given new "DeletePipeline" request

‎services/observability_pipelines/src/v2/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ export { ObservabilityPipelineDatadogTagsProcessorMode } from "./models/Observab
7272
export { ObservabilityPipelineDatadogTagsProcessorType } from "./models/ObservabilityPipelineDatadogTagsProcessorType";
7373
export { ObservabilityPipelineDecoding } from "./models/ObservabilityPipelineDecoding";
7474
export { ObservabilityPipelineDedupeProcessor } from "./models/ObservabilityPipelineDedupeProcessor";
75+
export { ObservabilityPipelineDedupeProcessorCache } from "./models/ObservabilityPipelineDedupeProcessorCache";
7576
export { ObservabilityPipelineDedupeProcessorMode } from "./models/ObservabilityPipelineDedupeProcessorMode";
7677
export { ObservabilityPipelineDedupeProcessorType } from "./models/ObservabilityPipelineDedupeProcessorType";
7778
export { ObservabilityPipelineDiskBufferOptions } from "./models/ObservabilityPipelineDiskBufferOptions";

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineDedupeProcessor.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { AttributeTypeMap } from "@datadog/datadog-api-client";
22

3+
import { ObservabilityPipelineDedupeProcessorCache } from "./ObservabilityPipelineDedupeProcessorCache";
34
import { ObservabilityPipelineDedupeProcessorMode } from "./ObservabilityPipelineDedupeProcessorMode";
45
import { ObservabilityPipelineDedupeProcessorType } from "./ObservabilityPipelineDedupeProcessorType";
56

@@ -9,6 +10,10 @@ import { ObservabilityPipelineDedupeProcessorType } from "./ObservabilityPipelin
910
* **Supported pipeline types:** logs
1011
*/
1112
export class ObservabilityPipelineDedupeProcessor {
13+
/**
14+
* Configuration for the cache used to detect duplicates.
15+
*/
16+
"cache"?: ObservabilityPipelineDedupeProcessorCache;
1217
/**
1318
* The display name for a component.
1419
*/
@@ -52,6 +57,10 @@ export class ObservabilityPipelineDedupeProcessor {
5257
* @ignore
5358
*/
5459
static readonly attributeTypeMap: AttributeTypeMap = {
60+
cache: {
61+
baseName: "cache",
62+
type: "ObservabilityPipelineDedupeProcessorCache",
63+
},
5564
displayName: {
5665
baseName: "display_name",
5766
type: "string",

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineDedupeProcessorCache.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import { AttributeTypeMap } from "@datadog/datadog-api-client";
2+
3+
/**
4+
* Configuration for the cache used to detect duplicates.
5+
*/
6+
export class ObservabilityPipelineDedupeProcessorCache {
7+
/**
8+
* The number of events to cache for duplicate detection.
9+
*/
10+
"numEvents": number;
11+
/**
12+
* A container for additional, undeclared properties.
13+
* This is a holder for any undeclared properties as specified with
14+
* the 'additionalProperties' keyword in the OAS document.
15+
*/
16+
"additionalProperties"?: { [key: string]: any };
17+
/**
18+
* @ignore
19+
*/
20+
"_unparsed"?: boolean;
21+
22+
/**
23+
* @ignore
24+
*/
25+
static readonly attributeTypeMap: AttributeTypeMap = {
26+
numEvents: {
27+
baseName: "num_events",
28+
type: "number",
29+
required: true,
30+
format: "int64",
31+
},
32+
additionalProperties: {
33+
baseName: "additionalProperties",
34+
type: "{ [key: string]: any; }",
35+
},
36+
};
37+
38+
/**
39+
* @ignore
40+
*/
41+
static getAttributeTypeMap(): AttributeTypeMap {
42+
return ObservabilityPipelineDedupeProcessorCache.attributeTypeMap;
43+
}
44+
45+
public constructor() {}
46+
}

‎services/observability_pipelines/src/v2/models/TypingInfo.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import { ObservabilityPipelineDatadogLogsDestinationRoute } from "./Observabilit
3232
import { ObservabilityPipelineDatadogMetricsDestination } from "./ObservabilityPipelineDatadogMetricsDestination";
3333
import { ObservabilityPipelineDatadogTagsProcessor } from "./ObservabilityPipelineDatadogTagsProcessor";
3434
import { ObservabilityPipelineDedupeProcessor } from "./ObservabilityPipelineDedupeProcessor";
35+
import { ObservabilityPipelineDedupeProcessorCache } from "./ObservabilityPipelineDedupeProcessorCache";
3536
import { ObservabilityPipelineDiskBufferOptions } from "./ObservabilityPipelineDiskBufferOptions";
3637
import { ObservabilityPipelineElasticsearchDestination } from "./ObservabilityPipelineElasticsearchDestination";
3738
import { ObservabilityPipelineElasticsearchDestinationDataStream } from "./ObservabilityPipelineElasticsearchDestinationDataStream";
@@ -570,6 +571,8 @@ export const TypingInfo: ModelTypingInfo = {
570571
ObservabilityPipelineDatadogTagsProcessor:
571572
ObservabilityPipelineDatadogTagsProcessor,
572573
ObservabilityPipelineDedupeProcessor: ObservabilityPipelineDedupeProcessor,
574+
ObservabilityPipelineDedupeProcessorCache:
575+
ObservabilityPipelineDedupeProcessorCache,
573576
ObservabilityPipelineDiskBufferOptions:
574577
ObservabilityPipelineDiskBufferOptions,
575578
ObservabilityPipelineElasticsearchDestination:

0 commit comments

Comments
 (0)