Skip to content

Commit 4429d82

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for cache property in dedupe processor in observability pipelines (DataDog#3690)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 71eea23 commit 4429d82

10 files changed

Lines changed: 427 additions & 8 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39514,6 +39514,8 @@ components:
3951439514

3951539515
**Supported pipeline types:** logs'
3951639516
properties:
39517+
cache:
39518+
$ref: '#/components/schemas/ObservabilityPipelineDedupeProcessorCache'
3951739519
display_name:
3951839520
$ref: '#/components/schemas/ObservabilityPipelineComponentDisplayName'
3951939521
enabled:
@@ -39551,6 +39553,19 @@ components:
3955139553
type: object
3955239554
x-pipeline-types:
3955339555
- logs
39556+
ObservabilityPipelineDedupeProcessorCache:
39557+
description: Configuration for the cache used to detect duplicates.
39558+
properties:
39559+
num_events:
39560+
description: The number of events to cache for duplicate detection.
39561+
example: 5000
39562+
format: int64
39563+
maximum: 1000000000
39564+
minimum: 1
39565+
type: integer
39566+
required:
39567+
- num_events
39568+
type: object
3955439569
ObservabilityPipelineDedupeProcessorMode:
3955539570
description: The deduplication mode to apply to the fields.
3955639571
enum:

‎api/datadogV2/model_observability_pipeline_dedupe_processor.go‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
//
1515
// **Supported pipeline types:** logs
1616
type ObservabilityPipelineDedupeProcessor struct {
17+
// Configuration for the cache used to detect duplicates.
18+
Cache *ObservabilityPipelineDedupeProcessorCache `json:"cache,omitempty"`
1719
// The display name for a component.
1820
DisplayName *string `json:"display_name,omitempty"`
1921
// Indicates whether the processor is enabled.
@@ -58,6 +60,34 @@ func NewObservabilityPipelineDedupeProcessorWithDefaults() *ObservabilityPipelin
5860
return &this
5961
}
6062

63+
// GetCache returns the Cache field value if set, zero value otherwise.
64+
func (o *ObservabilityPipelineDedupeProcessor) GetCache() ObservabilityPipelineDedupeProcessorCache {
65+
if o == nil || o.Cache == nil {
66+
var ret ObservabilityPipelineDedupeProcessorCache
67+
return ret
68+
}
69+
return *o.Cache
70+
}
71+
72+
// GetCacheOk returns a tuple with the Cache field value if set, nil otherwise
73+
// and a boolean to check if the value has been set.
74+
func (o *ObservabilityPipelineDedupeProcessor) GetCacheOk() (*ObservabilityPipelineDedupeProcessorCache, bool) {
75+
if o == nil || o.Cache == nil {
76+
return nil, false
77+
}
78+
return o.Cache, true
79+
}
80+
81+
// HasCache returns a boolean if a field has been set.
82+
func (o *ObservabilityPipelineDedupeProcessor) HasCache() bool {
83+
return o != nil && o.Cache != nil
84+
}
85+
86+
// SetCache gets a reference to the given ObservabilityPipelineDedupeProcessorCache and assigns it to the Cache field.
87+
func (o *ObservabilityPipelineDedupeProcessor) SetCache(v ObservabilityPipelineDedupeProcessorCache) {
88+
o.Cache = &v
89+
}
90+
6191
// GetDisplayName returns the DisplayName field value if set, zero value otherwise.
6292
func (o *ObservabilityPipelineDedupeProcessor) GetDisplayName() string {
6393
if o == nil || o.DisplayName == nil {
@@ -230,6 +260,9 @@ func (o ObservabilityPipelineDedupeProcessor) MarshalJSON() ([]byte, error) {
230260
if o.UnparsedObject != nil {
231261
return datadog.Marshal(o.UnparsedObject)
232262
}
263+
if o.Cache != nil {
264+
toSerialize["cache"] = o.Cache
265+
}
233266
if o.DisplayName != nil {
234267
toSerialize["display_name"] = o.DisplayName
235268
}
@@ -249,13 +282,14 @@ func (o ObservabilityPipelineDedupeProcessor) MarshalJSON() ([]byte, error) {
249282
// UnmarshalJSON deserializes the given payload.
250283
func (o *ObservabilityPipelineDedupeProcessor) UnmarshalJSON(bytes []byte) (err error) {
251284
all := struct {
252-
DisplayName *string `json:"display_name,omitempty"`
253-
Enabled *bool `json:"enabled"`
254-
Fields *[]string `json:"fields"`
255-
Id *string `json:"id"`
256-
Include *string `json:"include"`
257-
Mode *ObservabilityPipelineDedupeProcessorMode `json:"mode"`
258-
Type *ObservabilityPipelineDedupeProcessorType `json:"type"`
285+
Cache *ObservabilityPipelineDedupeProcessorCache `json:"cache,omitempty"`
286+
DisplayName *string `json:"display_name,omitempty"`
287+
Enabled *bool `json:"enabled"`
288+
Fields *[]string `json:"fields"`
289+
Id *string `json:"id"`
290+
Include *string `json:"include"`
291+
Mode *ObservabilityPipelineDedupeProcessorMode `json:"mode"`
292+
Type *ObservabilityPipelineDedupeProcessorType `json:"type"`
259293
}{}
260294
if err = datadog.Unmarshal(bytes, &all); err != nil {
261295
return datadog.Unmarshal(bytes, &o.UnparsedObject)
@@ -280,12 +314,16 @@ func (o *ObservabilityPipelineDedupeProcessor) UnmarshalJSON(bytes []byte) (err
280314
}
281315
additionalProperties := make(map[string]interface{})
282316
if err = datadog.Unmarshal(bytes, &additionalProperties); err == nil {
283-
datadog.DeleteKeys(additionalProperties, &[]string{"display_name", "enabled", "fields", "id", "include", "mode", "type"})
317+
datadog.DeleteKeys(additionalProperties, &[]string{"cache", "display_name", "enabled", "fields", "id", "include", "mode", "type"})
284318
} else {
285319
return err
286320
}
287321

288322
hasInvalidField := false
323+
if all.Cache != nil && all.Cache.UnparsedObject != nil && o.UnparsedObject == nil {
324+
hasInvalidField = true
325+
}
326+
o.Cache = all.Cache
289327
o.DisplayName = all.DisplayName
290328
o.Enabled = *all.Enabled
291329
o.Fields = *all.Fields

‎api/datadogV2/model_observability_pipeline_dedupe_processor_cache.go‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License.
2+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
3+
// Copyright 2019-Present Datadog, Inc.
4+
5+
package datadogV2
6+
7+
import (
8+
"fmt"
9+
10+
"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
11+
)
12+
13+
// ObservabilityPipelineDedupeProcessorCache Configuration for the cache used to detect duplicates.
14+
type ObservabilityPipelineDedupeProcessorCache struct {
15+
// The number of events to cache for duplicate detection.
16+
NumEvents int64 `json:"num_events"`
17+
// UnparsedObject contains the raw value of the object if there was an error when deserializing into the struct
18+
UnparsedObject map[string]interface{} `json:"-"`
19+
AdditionalProperties map[string]interface{} `json:"-"`
20+
}
21+
22+
// NewObservabilityPipelineDedupeProcessorCache instantiates a new ObservabilityPipelineDedupeProcessorCache object.
23+
// This constructor will assign default values to properties that have it defined,
24+
// and makes sure properties required by API are set, but the set of arguments
25+
// will change when the set of required properties is changed.
26+
func NewObservabilityPipelineDedupeProcessorCache(numEvents int64) *ObservabilityPipelineDedupeProcessorCache {
27+
this := ObservabilityPipelineDedupeProcessorCache{}
28+
this.NumEvents = numEvents
29+
return &this
30+
}
31+
32+
// NewObservabilityPipelineDedupeProcessorCacheWithDefaults instantiates a new ObservabilityPipelineDedupeProcessorCache object.
33+
// This constructor will only assign default values to properties that have it defined,
34+
// but it doesn't guarantee that properties required by API are set.
35+
func NewObservabilityPipelineDedupeProcessorCacheWithDefaults() *ObservabilityPipelineDedupeProcessorCache {
36+
this := ObservabilityPipelineDedupeProcessorCache{}
37+
return &this
38+
}
39+
40+
// GetNumEvents returns the NumEvents field value.
41+
func (o *ObservabilityPipelineDedupeProcessorCache) GetNumEvents() int64 {
42+
if o == nil {
43+
var ret int64
44+
return ret
45+
}
46+
return o.NumEvents
47+
}
48+
49+
// GetNumEventsOk returns a tuple with the NumEvents field value
50+
// and a boolean to check if the value has been set.
51+
func (o *ObservabilityPipelineDedupeProcessorCache) GetNumEventsOk() (*int64, bool) {
52+
if o == nil {
53+
return nil, false
54+
}
55+
return &o.NumEvents, true
56+
}
57+
58+
// SetNumEvents sets field value.
59+
func (o *ObservabilityPipelineDedupeProcessorCache) SetNumEvents(v int64) {
60+
o.NumEvents = v
61+
}
62+
63+
// MarshalJSON serializes the struct using spec logic.
64+
func (o ObservabilityPipelineDedupeProcessorCache) MarshalJSON() ([]byte, error) {
65+
toSerialize := map[string]interface{}{}
66+
if o.UnparsedObject != nil {
67+
return datadog.Marshal(o.UnparsedObject)
68+
}
69+
toSerialize["num_events"] = o.NumEvents
70+
71+
for key, value := range o.AdditionalProperties {
72+
toSerialize[key] = value
73+
}
74+
return datadog.Marshal(toSerialize)
75+
}
76+
77+
// UnmarshalJSON deserializes the given payload.
78+
func (o *ObservabilityPipelineDedupeProcessorCache) UnmarshalJSON(bytes []byte) (err error) {
79+
all := struct {
80+
NumEvents *int64 `json:"num_events"`
81+
}{}
82+
if err = datadog.Unmarshal(bytes, &all); err != nil {
83+
return datadog.Unmarshal(bytes, &o.UnparsedObject)
84+
}
85+
if all.NumEvents == nil {
86+
return fmt.Errorf("required field num_events missing")
87+
}
88+
additionalProperties := make(map[string]interface{})
89+
if err = datadog.Unmarshal(bytes, &additionalProperties); err == nil {
90+
datadog.DeleteKeys(additionalProperties, &[]string{"num_events"})
91+
} else {
92+
return err
93+
}
94+
o.NumEvents = *all.NumEvents
95+
96+
if len(additionalProperties) > 0 {
97+
o.AdditionalProperties = additionalProperties
98+
}
99+
100+
return nil
101+
}

‎examples/v2/observability-pipelines/CreatePipeline_3363445359.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Create a pipeline with dedupe processor with cache returns "OK" response
2+
3+
package main
4+
5+
import (
6+
"context"
7+
"encoding/json"
8+
"fmt"
9+
"os"
10+
11+
"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
12+
"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
13+
)
14+
15+
func main() {
16+
body := datadogV2.ObservabilityPipelineSpec{
17+
Data: datadogV2.ObservabilityPipelineSpecData{
18+
Attributes: datadogV2.ObservabilityPipelineDataAttributes{
19+
Config: datadogV2.ObservabilityPipelineConfig{
20+
Destinations: []datadogV2.ObservabilityPipelineConfigDestinationItem{
21+
datadogV2.ObservabilityPipelineConfigDestinationItem{
22+
ObservabilityPipelineDatadogLogsDestination: &datadogV2.ObservabilityPipelineDatadogLogsDestination{
23+
Id: "datadog-logs-destination",
24+
Inputs: []string{
25+
"my-processor-group",
26+
},
27+
Type: datadogV2.OBSERVABILITYPIPELINEDATADOGLOGSDESTINATIONTYPE_DATADOG_LOGS,
28+
}},
29+
},
30+
ProcessorGroups: []datadogV2.ObservabilityPipelineConfigProcessorGroup{
31+
{
32+
Enabled: true,
33+
Id: "my-processor-group",
34+
Include: "service:my-service",
35+
Inputs: []string{
36+
"datadog-agent-source",
37+
},
38+
Processors: []datadogV2.ObservabilityPipelineConfigProcessorItem{
39+
datadogV2.ObservabilityPipelineConfigProcessorItem{
40+
ObservabilityPipelineDedupeProcessor: &datadogV2.ObservabilityPipelineDedupeProcessor{
41+
Enabled: true,
42+
Id: "dedupe-processor",
43+
Include: "service:my-service",
44+
Type: datadogV2.OBSERVABILITYPIPELINEDEDUPEPROCESSORTYPE_DEDUPE,
45+
Fields: []string{
46+
"message",
47+
},
48+
Mode: datadogV2.OBSERVABILITYPIPELINEDEDUPEPROCESSORMODE_MATCH,
49+
Cache: &datadogV2.ObservabilityPipelineDedupeProcessorCache{
50+
NumEvents: 5000,
51+
},
52+
}},
53+
},
54+
},
55+
},
56+
Sources: []datadogV2.ObservabilityPipelineConfigSourceItem{
57+
datadogV2.ObservabilityPipelineConfigSourceItem{
58+
ObservabilityPipelineDatadogAgentSource: &datadogV2.ObservabilityPipelineDatadogAgentSource{
59+
Id: "datadog-agent-source",
60+
Type: datadogV2.OBSERVABILITYPIPELINEDATADOGAGENTSOURCETYPE_DATADOG_AGENT,
61+
}},
62+
},
63+
},
64+
Name: "Pipeline with Dedupe Cache",
65+
},
66+
Type: "pipelines",
67+
},
68+
}
69+
ctx := datadog.NewDefaultContext(context.Background())
70+
configuration := datadog.NewConfiguration()
71+
apiClient := datadog.NewAPIClient(configuration)
72+
api := datadogV2.NewObservabilityPipelinesApi(apiClient)
73+
resp, r, err := api.CreatePipeline(ctx, body)
74+
75+
if err != nil {
76+
fmt.Fprintf(os.Stderr, "Error when calling `ObservabilityPipelinesApi.CreatePipeline`: %v\n", err)
77+
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
78+
}
79+
80+
responseContent, _ := json.MarshalIndent(resp, "", " ")
81+
fmt.Fprintf(os.Stdout, "Response from `ObservabilityPipelinesApi.CreatePipeline`:\n%s\n", responseContent)
82+
}

‎examples/v2/observability-pipelines/CreatePipeline_581245895.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Create a pipeline with dedupe processor without cache returns "OK" response
2+
3+
package main
4+
5+
import (
6+
"context"
7+
"encoding/json"
8+
"fmt"
9+
"os"
10+
11+
"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
12+
"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
13+
)
14+
15+
func main() {
16+
body := datadogV2.ObservabilityPipelineSpec{
17+
Data: datadogV2.ObservabilityPipelineSpecData{
18+
Attributes: datadogV2.ObservabilityPipelineDataAttributes{
19+
Config: datadogV2.ObservabilityPipelineConfig{
20+
Destinations: []datadogV2.ObservabilityPipelineConfigDestinationItem{
21+
datadogV2.ObservabilityPipelineConfigDestinationItem{
22+
ObservabilityPipelineDatadogLogsDestination: &datadogV2.ObservabilityPipelineDatadogLogsDestination{
23+
Id: "datadog-logs-destination",
24+
Inputs: []string{
25+
"my-processor-group",
26+
},
27+
Type: datadogV2.OBSERVABILITYPIPELINEDATADOGLOGSDESTINATIONTYPE_DATADOG_LOGS,
28+
}},
29+
},
30+
ProcessorGroups: []datadogV2.ObservabilityPipelineConfigProcessorGroup{
31+
{
32+
Enabled: true,
33+
Id: "my-processor-group",
34+
Include: "service:my-service",
35+
Inputs: []string{
36+
"datadog-agent-source",
37+
},
38+
Processors: []datadogV2.ObservabilityPipelineConfigProcessorItem{
39+
datadogV2.ObservabilityPipelineConfigProcessorItem{
40+
ObservabilityPipelineDedupeProcessor: &datadogV2.ObservabilityPipelineDedupeProcessor{
41+
Enabled: true,
42+
Id: "dedupe-processor",
43+
Include: "service:my-service",
44+
Type: datadogV2.OBSERVABILITYPIPELINEDEDUPEPROCESSORTYPE_DEDUPE,
45+
Fields: []string{
46+
"message",
47+
},
48+
Mode: datadogV2.OBSERVABILITYPIPELINEDEDUPEPROCESSORMODE_MATCH,
49+
}},
50+
},
51+
},
52+
},
53+
Sources: []datadogV2.ObservabilityPipelineConfigSourceItem{
54+
datadogV2.ObservabilityPipelineConfigSourceItem{
55+
ObservabilityPipelineDatadogAgentSource: &datadogV2.ObservabilityPipelineDatadogAgentSource{
56+
Id: "datadog-agent-source",
57+
Type: datadogV2.OBSERVABILITYPIPELINEDATADOGAGENTSOURCETYPE_DATADOG_AGENT,
58+
}},
59+
},
60+
},
61+
Name: "Pipeline with Dedupe No Cache",
62+
},
63+
Type: "pipelines",
64+
},
65+
}
66+
ctx := datadog.NewDefaultContext(context.Background())
67+
configuration := datadog.NewConfiguration()
68+
apiClient := datadog.NewAPIClient(configuration)
69+
api := datadogV2.NewObservabilityPipelinesApi(apiClient)
70+
resp, r, err := api.CreatePipeline(ctx, body)
71+
72+
if err != nil {
73+
fmt.Fprintf(os.Stderr, "Error when calling `ObservabilityPipelinesApi.CreatePipeline`: %v\n", err)
74+
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
75+
}
76+
77+
responseContent, _ := json.MarshalIndent(resp, "", " ")
78+
fmt.Fprintf(os.Stdout, "Response from `ObservabilityPipelinesApi.CreatePipeline`:\n%s\n", responseContent)
79+
}

‎tests/scenarios/cassettes/TestScenarios/v2/Feature_Observability_Pipelines/Scenario_Create_a_pipeline_with_dedupe_processor_with_cache_returns_OK_response.freeze‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
2026-02-09T09:53:31.840Z

0 commit comments

Comments
 (0)