Skip to content

Commit 7c443c5

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Adding all Observability Pipeline new hec_token related features into the api (#3894)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent a159056 commit 7c443c5

File tree

18 files changed

+500
-6
lines changed

18 files changed

+500
-6
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44467,6 +44467,40 @@ components:
4446744467
type: string
4446844468
x-enum-varnames:
4446944469
- ELASTICSEARCH
44470+
ObservabilityPipelineEnrichmentTableFieldEventLookup:
44471+
description: Looks up a value from a field path in the log event.
44472+
properties:
44473+
event:
44474+
description: The path to the field in the log event to use as the lookup key.
44475+
example: log.user.id
44476+
type: string
44477+
required:
44478+
- event
44479+
type: object
44480+
ObservabilityPipelineEnrichmentTableFieldSecretLookup:
44481+
description: Looks up a value stored as a pipeline secret.
44482+
properties:
44483+
secret:
44484+
description: The name of the secret containing the lookup key value.
44485+
example: MY_LOOKUP_SECRET
44486+
type: string
44487+
required:
44488+
- secret
44489+
type: object
44490+
ObservabilityPipelineEnrichmentTableFieldStringPath:
44491+
description: A plain field path in the log event, used as the lookup key.
44492+
example: log.user.id
44493+
type: string
44494+
ObservabilityPipelineEnrichmentTableFieldVrlLookup:
44495+
description: Evaluates a VRL expression to produce the lookup key.
44496+
properties:
44497+
vrl:
44498+
description: A VRL expression that returns the value to use as the lookup key.
44499+
example: .log.user.id
44500+
type: string
44501+
required:
44502+
- vrl
44503+
type: object
4447044504
ObservabilityPipelineEnrichmentTableFile:
4447144505
description: Defines a static enrichment table loaded from a CSV file.
4447244506
properties:
@@ -44517,6 +44551,16 @@ components:
4451744551
type: string
4451844552
x-enum-varnames:
4451944553
- CSV
44554+
ObservabilityPipelineEnrichmentTableFileKeyItemField:
44555+
description: |-
44556+
Specifies the source of the key value used for enrichment table lookups.
44557+
Can be a plain field path string or an object specifying `event`, `vrl`, or `secret`.
44558+
example: log.user.id
44559+
oneOf:
44560+
- $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFieldStringPath"
44561+
- $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFieldEventLookup"
44562+
- $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFieldVrlLookup"
44563+
- $ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFieldSecretLookup"
4452044564
ObservabilityPipelineEnrichmentTableFileKeyItems:
4452144565
description: Defines how to map log fields to enrichment table columns during lookups.
4452244566
properties:
@@ -44527,9 +44571,7 @@ components:
4452744571
comparison:
4452844572
$ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFileKeyItemsComparison"
4452944573
field:
44530-
description: The `items` `field`.
44531-
example: log.user.id
44532-
type: string
44574+
$ref: "#/components/schemas/ObservabilityPipelineEnrichmentTableFileKeyItemField"
4453344575
required:
4453444576
- column
4453544577
- comparison
@@ -47550,6 +47592,8 @@ components:
4755047592
description: Name of the environment variable or secret that holds the Splunk HEC token.
4755147593
example: SPLUNK_HEC_TOKEN
4755247594
type: string
47595+
token_strategy:
47596+
$ref: "#/components/schemas/ObservabilityPipelineSplunkHecDestinationTokenStrategy"
4755347597
type:
4755447598
$ref: "#/components/schemas/ObservabilityPipelineSplunkHecDestinationType"
4755547599
required:
@@ -47568,6 +47612,16 @@ components:
4756847612
x-enum-varnames:
4756947613
- JSON
4757047614
- RAW_MESSAGE
47615+
ObservabilityPipelineSplunkHecDestinationTokenStrategy:
47616+
description: Controls how the Splunk HEC token is supplied. Use `custom` to provide a token with `token_key`, or `from_source` to forward the token received from an upstream Splunk HEC source.
47617+
enum:
47618+
- custom
47619+
- from_source
47620+
example: custom
47621+
type: string
47622+
x-enum-varnames:
47623+
- CUSTOM
47624+
- FROM_SOURCE
4757147625
ObservabilityPipelineSplunkHecDestinationType:
4757247626
default: splunk_hec
4757347627
description: The destination type. Always `splunk_hec`.
@@ -47591,6 +47645,12 @@ components:
4759147645
description: The unique identifier for this component. Used in other parts of the pipeline to reference this component (for example, as the `input` to downstream components).
4759247646
example: splunk-hec-source
4759347647
type: string
47648+
store_hec_token:
47649+
description: |-
47650+
When `true`, the Splunk HEC token from the incoming request is stored in the event metadata.
47651+
This allows downstream components to forward the token to other Splunk HEC destinations.
47652+
example: true
47653+
type: boolean
4759447654
tls:
4759547655
$ref: "#/components/schemas/ObservabilityPipelineTls"
4759647656
type:

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-Splunk-HEC-destination-token_strategy-returns-OK-r_276364485/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-04-08T15:11:59.762Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-Splunk-HEC-destination-token_strategy-returns-OK-r_276364485/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with Splunk HEC destination token_strategy returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "71464a8fa8e1502453de0fde42aaea8c",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 553,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"splunk-hec-destination\",\"inputs\":[\"my-processor-group\"],\"token_key\":\"SPLUNK_HEC_TOKEN\",\"token_strategy\":\"custom\",\"type\":\"splunk_hec\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with Splunk HEC token_strategy\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-04-08T15:11:59.764Z",
61+
"time": 370
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-Splunk-HEC-source-store_hec_token-returns-OK-respo_247742775/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-04-08T15:11:59.370Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-Splunk-HEC-source-store_hec_token-returns-OK-respo_247742775/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with Splunk HEC source store_hec_token returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "af5cae27d9df78c907ef82b805cf3c0f",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 515,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"splunk-hec-source\"],\"processors\":[{\"enabled\":true,\"id\":\"filter-processor\",\"include\":\"status:error\",\"type\":\"filter\"}]}],\"sources\":[{\"id\":\"splunk-hec-source\",\"store_hec_token\":true,\"type\":\"splunk_hec\"}]},\"name\":\"Pipeline with Splunk HEC store_hec_token\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 369,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-04-08T15:11:59.373Z",
61+
"time": 383
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-enrichment-table-secret-field-lookup-returns-OK-resp_5418750/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-04-09T11:54:29.220Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-enrichment-table-secret-field-lookup-returns-OK-resp_5418750/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with enrichment table secret field lookup returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "862a8010da15d364489e4b7b8dc6e8ba",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 783,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"file\":{\"encoding\":{\"delimiter\":\",\",\"includes_headers\":true,\"type\":\"csv\"},\"key\":[{\"column\":\"user_id\",\"comparison\":\"equals\",\"field\":{\"secret\":\"LOOKUP_KEY_SECRET\"}}],\"path\":\"/etc/enrichment/lookup.csv\",\"schema\":[{\"column\":\"user_id\",\"type\":\"string\"}]},\"id\":\"enrichment-processor\",\"include\":\"*\",\"target\":\"enriched\",\"type\":\"enrichment_table\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"Pipeline with Enrichment Table Secret Field Lookup\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-04-09T11:54:30.102Z",
61+
"time": 216
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎features/v2/observability_pipelines.feature‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,22 @@ Feature: Observability Pipelines
207207
Then the response status is 200 OK
208208
And the response "errors" has length 0
209209

210+
@team:DataDog/observability-pipelines
211+
Scenario: Validate an observability pipeline with Splunk HEC destination token_strategy returns "OK" response
212+
Given new "ValidatePipeline" request
213+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "splunk-hec-destination", "inputs": ["my-processor-group"], "type": "splunk_hec", "token_key": "SPLUNK_HEC_TOKEN", "token_strategy": "custom"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Splunk HEC token_strategy"}, "type": "pipelines"}}
214+
When the request is sent
215+
Then the response status is 200 OK
216+
And the response "errors" has length 0
217+
218+
@team:DataDog/observability-pipelines
219+
Scenario: Validate an observability pipeline with Splunk HEC source store_hec_token returns "OK" response
220+
Given new "ValidatePipeline" request
221+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["splunk-hec-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "splunk-hec-source", "type": "splunk_hec", "store_hec_token": true}]}, "name": "Pipeline with Splunk HEC store_hec_token"}, "type": "pipelines"}}
222+
When the request is sent
223+
Then the response status is 200 OK
224+
And the response "errors" has length 0
225+
210226
@team:DataDog/observability-pipelines
211227
Scenario: Validate an observability pipeline with amazon S3 source compression returns "OK" response
212228
Given new "ValidatePipeline" request
@@ -223,6 +239,14 @@ Feature: Observability Pipelines
223239
Then the response status is 200 OK
224240
And the response "errors" has length 0
225241

242+
@team:DataDog/observability-pipelines
243+
Scenario: Validate an observability pipeline with enrichment table secret field lookup returns "OK" response
244+
Given new "ValidatePipeline" request
245+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "enrichment-processor", "include": "*", "target": "enriched", "type": "enrichment_table", "file": {"encoding": {"delimiter": ",", "type": "csv", "includes_headers": true}, "key": [{"column": "user_id", "comparison": "equals", "field": {"secret": "LOOKUP_KEY_SECRET"}}], "path": "/etc/enrichment/lookup.csv", "schema": [{"column": "user_id", "type": "string"}]}}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Enrichment Table Secret Field Lookup"}, "type": "pipelines"}}
246+
When the request is sent
247+
Then the response status is 200 OK
248+
And the response "errors" has length 0
249+
226250
@team:DataDog/observability-pipelines
227251
Scenario: Validate an observability pipeline with source secret key returns "OK" response
228252
Given new "ValidatePipeline" request

‎services/observability_pipelines/src/v2/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,13 @@ export { ObservabilityPipelineElasticsearchDestinationCompression } from "./mode
9999
export { ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm } from "./models/ObservabilityPipelineElasticsearchDestinationCompressionAlgorithm";
100100
export { ObservabilityPipelineElasticsearchDestinationDataStream } from "./models/ObservabilityPipelineElasticsearchDestinationDataStream";
101101
export { ObservabilityPipelineElasticsearchDestinationType } from "./models/ObservabilityPipelineElasticsearchDestinationType";
102+
export { ObservabilityPipelineEnrichmentTableFieldEventLookup } from "./models/ObservabilityPipelineEnrichmentTableFieldEventLookup";
103+
export { ObservabilityPipelineEnrichmentTableFieldSecretLookup } from "./models/ObservabilityPipelineEnrichmentTableFieldSecretLookup";
104+
export { ObservabilityPipelineEnrichmentTableFieldVrlLookup } from "./models/ObservabilityPipelineEnrichmentTableFieldVrlLookup";
102105
export { ObservabilityPipelineEnrichmentTableFile } from "./models/ObservabilityPipelineEnrichmentTableFile";
103106
export { ObservabilityPipelineEnrichmentTableFileEncoding } from "./models/ObservabilityPipelineEnrichmentTableFileEncoding";
104107
export { ObservabilityPipelineEnrichmentTableFileEncodingType } from "./models/ObservabilityPipelineEnrichmentTableFileEncodingType";
108+
export { ObservabilityPipelineEnrichmentTableFileKeyItemField } from "./models/ObservabilityPipelineEnrichmentTableFileKeyItemField";
105109
export { ObservabilityPipelineEnrichmentTableFileKeyItems } from "./models/ObservabilityPipelineEnrichmentTableFileKeyItems";
106110
export { ObservabilityPipelineEnrichmentTableFileKeyItemsComparison } from "./models/ObservabilityPipelineEnrichmentTableFileKeyItemsComparison";
107111
export { ObservabilityPipelineEnrichmentTableFileSchemaItems } from "./models/ObservabilityPipelineEnrichmentTableFileSchemaItems";
@@ -282,6 +286,7 @@ export { ObservabilityPipelineSplitArrayProcessorArrayConfig } from "./models/Ob
282286
export { ObservabilityPipelineSplitArrayProcessorType } from "./models/ObservabilityPipelineSplitArrayProcessorType";
283287
export { ObservabilityPipelineSplunkHecDestination } from "./models/ObservabilityPipelineSplunkHecDestination";
284288
export { ObservabilityPipelineSplunkHecDestinationEncoding } from "./models/ObservabilityPipelineSplunkHecDestinationEncoding";
289+
export { ObservabilityPipelineSplunkHecDestinationTokenStrategy } from "./models/ObservabilityPipelineSplunkHecDestinationTokenStrategy";
285290
export { ObservabilityPipelineSplunkHecDestinationType } from "./models/ObservabilityPipelineSplunkHecDestinationType";
286291
export { ObservabilityPipelineSplunkHecSource } from "./models/ObservabilityPipelineSplunkHecSource";
287292
export { ObservabilityPipelineSplunkHecSourceType } from "./models/ObservabilityPipelineSplunkHecSourceType";

0 commit comments

Comments
 (0)