Skip to content

Commit e842713

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Adding custom mapper support to Observability Pipelines OCSF Mapper (#3455)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent ec49872 commit e842713

File tree

16 files changed

+715
-0
lines changed

16 files changed

+715
-0
lines changed

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41369,6 +41369,7 @@ components:
4136941369
example: CloudTrail Account Change
4137041370
oneOf:
4137141371
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingLibrary'
41372+
- $ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustom'
4137241373
ObservabilityPipelineOcsfMapperProcessorType:
4137341374
default: ocsf_mapper
4137441375
description: The processor type. The value should always be `ocsf_mapper`.
@@ -41378,6 +41379,116 @@ components:
4137841379
type: string
4137941380
x-enum-varnames:
4138041381
- OCSF_MAPPER
41382+
ObservabilityPipelineOcsfMappingCustom:
41383+
description: Custom OCSF mapping configuration for transforming logs.
41384+
properties:
41385+
mapping:
41386+
description: A list of field mapping rules for transforming log fields to
41387+
OCSF schema fields.
41388+
items:
41389+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomFieldMapping'
41390+
type: array
41391+
metadata:
41392+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomMetadata'
41393+
version:
41394+
description: The version of the custom mapping configuration.
41395+
example: 1
41396+
format: int64
41397+
type: integer
41398+
required:
41399+
- mapping
41400+
- metadata
41401+
- version
41402+
type: object
41403+
ObservabilityPipelineOcsfMappingCustomFieldMapping:
41404+
description: Defines a single field mapping rule for transforming a source field
41405+
to an OCSF destination field.
41406+
properties:
41407+
default:
41408+
description: The default value to use if the source field is missing or
41409+
empty.
41410+
example: ''
41411+
dest:
41412+
description: The destination OCSF field path.
41413+
example: device.type
41414+
type: string
41415+
lookup:
41416+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookup'
41417+
source:
41418+
description: The source field path from the log event.
41419+
example: host.type
41420+
sources:
41421+
description: Multiple source field paths for combined mapping.
41422+
example:
41423+
- field1
41424+
- field2
41425+
value:
41426+
description: A static value to use for the destination field.
41427+
example: static_value
41428+
required:
41429+
- dest
41430+
type: object
41431+
ObservabilityPipelineOcsfMappingCustomLookup:
41432+
description: Lookup table configuration for mapping source values to destination
41433+
values.
41434+
properties:
41435+
default:
41436+
description: The default value to use if no lookup match is found.
41437+
example: unknown
41438+
table:
41439+
description: A list of lookup table entries for value transformation.
41440+
items:
41441+
$ref: '#/components/schemas/ObservabilityPipelineOcsfMappingCustomLookupTableEntry'
41442+
type: array
41443+
type: object
41444+
ObservabilityPipelineOcsfMappingCustomLookupTableEntry:
41445+
description: A single entry in a lookup table for value transformation.
41446+
properties:
41447+
contains:
41448+
description: The substring to match in the source value.
41449+
example: Desktop
41450+
type: string
41451+
equals:
41452+
description: The exact value to match in the source.
41453+
example: desktop
41454+
equals_source:
41455+
description: The source field to match against.
41456+
example: device_type
41457+
type: string
41458+
matches:
41459+
description: A regex pattern to match in the source value.
41460+
example: ^Desktop.*
41461+
type: string
41462+
not_matches:
41463+
description: A regex pattern that must not match the source value.
41464+
example: ^Mobile.*
41465+
type: string
41466+
value:
41467+
description: The value to use when a match is found.
41468+
example: desktop
41469+
type: object
41470+
ObservabilityPipelineOcsfMappingCustomMetadata:
41471+
description: Metadata for the custom OCSF mapping.
41472+
properties:
41473+
class:
41474+
description: The OCSF event class name.
41475+
example: Device Inventory Info
41476+
type: string
41477+
profiles:
41478+
description: A list of OCSF profiles to apply.
41479+
example:
41480+
- container
41481+
items:
41482+
type: string
41483+
type: array
41484+
version:
41485+
description: The OCSF schema version.
41486+
example: 1.3.0
41487+
type: string
41488+
required:
41489+
- class
41490+
- version
41491+
type: object
4138141492
ObservabilityPipelineOcsfMappingLibrary:
4138241493
description: Predefined library mappings for common log formats.
4138341494
enum:

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-custom-mapping-returns-OK-response_1216017604/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-02-10T14:12:05.668Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-custom-mapping-returns-OK-response_1216017604/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with OCSF mapper custom mapping returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "53ece09f7d78b3327b7edee4b1d3bc51",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 888,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"ocsf-mapper-processor\",\"include\":\"service:my-service\",\"mappings\":[{\"include\":\"source:custom\",\"mapping\":{\"mapping\":[{\"default\":\"\",\"dest\":\"time\",\"source\":\"timestamp\"},{\"default\":\"\",\"dest\":\"severity\",\"source\":\"level\"},{\"default\":\"\",\"dest\":\"device.type\",\"lookup\":{\"table\":[{\"contains\":\"Desktop\",\"value\":\"desktop\"}]},\"source\":\"host.type\"}],\"metadata\":{\"class\":\"Device Inventory Info\",\"profiles\":[\"container\"],\"version\":\"1.3.0\"},\"version\":1}}],\"type\":\"ocsf_mapper\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"OCSF Custom Mapper Pipeline\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-02-10T14:12:05.673Z",
61+
"time": 381
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-invalid-custom-mapping-returns-Bad-Re_1912095981/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-02-10T14:12:06.064Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-invalid-custom-mapping-returns-Bad-Re_1912095981/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with OCSF mapper invalid custom mapping returns \"Bad Request\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "61f1236430793b647e3deccb484f7786",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 699,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"ocsf-mapper-processor\",\"include\":\"service:my-service\",\"mappings\":[{\"include\":\"source:custom\",\"mapping\":{\"mapping\":[{\"dest\":\"time\",\"source\":\"timestamp\"}],\"metadata\":{\"class\":\"Invalid Class\",\"profiles\":[\"container\"],\"version\":\"1.3.0\"},\"version\":0}}],\"type\":\"ocsf_mapper\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"OCSF Invalid Mapper Pipeline\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 344,
42+
"content": {
43+
"mimeType": "application/json",
44+
"size": 344,
45+
"text": "{\"errors\":[{\"title\":\"Schema version must be a positive integer\",\"meta\":{\"field\":\"mappings.0.version\",\"id\":\"ocsf-mapper-processor\",\"message\":\"Schema version must be a positive integer\"}},{\"title\":\"Invalid custom mapping class\",\"meta\":{\"field\":\"mappings.0.metadata.class\",\"id\":\"ocsf-mapper-processor\",\"message\":\"Invalid custom mapping class\"}}]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/json"
52+
}
53+
],
54+
"headersSize": 363,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 400,
58+
"statusText": "Bad Request"
59+
},
60+
"startedDateTime": "2026-02-10T14:12:06.069Z",
61+
"time": 372
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-library-mapping-returns-OK-response_935798470/frozen.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"2026-02-10T14:12:05.285Z"

‎cassettes/v2/Observability-Pipelines_4170000189/Validate-an-observability-pipeline-with-OCSF-mapper-library-mapping-returns-OK-response_935798470/recording.har‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"log": {
3+
"_recordingName": "Observability Pipelines/Validate an observability pipeline with OCSF mapper library mapping returns \"OK\" response",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.5"
8+
},
9+
"entries": [
10+
{
11+
"_id": "5e3a17b9b10ff8463285d7e228eca707",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 580,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"_fromType": "array",
20+
"name": "accept",
21+
"value": "application/json"
22+
},
23+
{
24+
"_fromType": "array",
25+
"name": "content-type",
26+
"value": "application/json"
27+
}
28+
],
29+
"headersSize": 583,
30+
"httpVersion": "HTTP/1.1",
31+
"method": "POST",
32+
"postData": {
33+
"mimeType": "application/json",
34+
"params": [],
35+
"text": "{\"data\":{\"attributes\":{\"config\":{\"destinations\":[{\"id\":\"datadog-logs-destination\",\"inputs\":[\"my-processor-group\"],\"type\":\"datadog_logs\"}],\"processor_groups\":[{\"enabled\":true,\"id\":\"my-processor-group\",\"include\":\"service:my-service\",\"inputs\":[\"datadog-agent-source\"],\"processors\":[{\"enabled\":true,\"id\":\"ocsf-mapper-processor\",\"include\":\"service:my-service\",\"mappings\":[{\"include\":\"source:cloudtrail\",\"mapping\":\"CloudTrail Account Change\"}],\"type\":\"ocsf_mapper\"}]}],\"sources\":[{\"id\":\"datadog-agent-source\",\"type\":\"datadog_agent\"}]},\"name\":\"OCSF Mapper Pipeline\"},\"type\":\"pipelines\"}}"
36+
},
37+
"queryString": [],
38+
"url": "https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/validate"
39+
},
40+
"response": {
41+
"bodySize": 14,
42+
"content": {
43+
"mimeType": "application/vnd.api+json",
44+
"size": 14,
45+
"text": "{\"errors\":[]}\n"
46+
},
47+
"cookies": [],
48+
"headers": [
49+
{
50+
"name": "content-type",
51+
"value": "application/vnd.api+json"
52+
}
53+
],
54+
"headersSize": 370,
55+
"httpVersion": "HTTP/1.1",
56+
"redirectURL": "",
57+
"status": 200,
58+
"statusText": "OK"
59+
},
60+
"startedDateTime": "2026-02-10T14:12:05.289Z",
61+
"time": 372
62+
}
63+
],
64+
"pages": [],
65+
"version": "1.2"
66+
}
67+
}

‎features/v2/observability_pipelines.feature‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,3 +167,26 @@ Feature: Observability Pipelines
167167
When the request is sent
168168
Then the response status is 200 OK
169169
And the response "errors" has length 0
170+
171+
@team:DataDog/observability-pipelines
172+
Scenario: Validate an observability pipeline with OCSF mapper custom mapping returns "OK" response
173+
Given new "ValidatePipeline" request
174+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "ocsf-mapper-processor", "include": "service:my-service", "mappings": [{"include": "source:custom", "mapping": {"mapping": [{"default": "", "dest": "time", "source": "timestamp"}, {"default": "", "dest": "severity", "source": "level"}, {"default": "", "dest": "device.type", "lookup": {"table": [{"contains": "Desktop", "value": "desktop"}]}, "source": "host.type"}], "metadata": {"class": "Device Inventory Info", "profiles": ["container"], "version": "1.3.0"}, "version": 1}}], "type": "ocsf_mapper"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "OCSF Custom Mapper Pipeline"}, "type": "pipelines"}}
175+
When the request is sent
176+
Then the response status is 200 OK
177+
And the response "errors" has length 0
178+
179+
@team:DataDog/observability-pipelines
180+
Scenario: Validate an observability pipeline with OCSF mapper invalid custom mapping returns "Bad Request" response
181+
Given new "ValidatePipeline" request
182+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "ocsf-mapper-processor", "include": "service:my-service", "mappings": [{"include": "source:custom", "mapping": {"mapping": [{"dest": "time", "source": "timestamp"}], "metadata": {"class": "Invalid Class", "profiles": ["container"], "version": "1.3.0"}, "version": 0}}], "type": "ocsf_mapper"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "OCSF Invalid Mapper Pipeline"}, "type": "pipelines"}}
183+
When the request is sent
184+
Then the response status is 400 Bad Request
185+
186+
@team:DataDog/observability-pipelines
187+
Scenario: Validate an observability pipeline with OCSF mapper library mapping returns "OK" response
188+
Given new "ValidatePipeline" request
189+
And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "ocsf-mapper-processor", "include": "service:my-service", "type": "ocsf_mapper", "mappings": [{"include": "source:cloudtrail", "mapping": "CloudTrail Account Change"}]}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "OCSF Mapper Pipeline"}, "type": "pipelines"}}
190+
When the request is sent
191+
Then the response status is 200 OK
192+
And the response "errors" has length 0

‎services/observability_pipelines/src/v2/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,11 @@ export { ObservabilityPipelineOcsfMapperProcessor } from "./models/Observability
158158
export { ObservabilityPipelineOcsfMapperProcessorMapping } from "./models/ObservabilityPipelineOcsfMapperProcessorMapping";
159159
export { ObservabilityPipelineOcsfMapperProcessorMappingMapping } from "./models/ObservabilityPipelineOcsfMapperProcessorMappingMapping";
160160
export { ObservabilityPipelineOcsfMapperProcessorType } from "./models/ObservabilityPipelineOcsfMapperProcessorType";
161+
export { ObservabilityPipelineOcsfMappingCustom } from "./models/ObservabilityPipelineOcsfMappingCustom";
162+
export { ObservabilityPipelineOcsfMappingCustomFieldMapping } from "./models/ObservabilityPipelineOcsfMappingCustomFieldMapping";
163+
export { ObservabilityPipelineOcsfMappingCustomLookup } from "./models/ObservabilityPipelineOcsfMappingCustomLookup";
164+
export { ObservabilityPipelineOcsfMappingCustomLookupTableEntry } from "./models/ObservabilityPipelineOcsfMappingCustomLookupTableEntry";
165+
export { ObservabilityPipelineOcsfMappingCustomMetadata } from "./models/ObservabilityPipelineOcsfMappingCustomMetadata";
161166
export { ObservabilityPipelineOcsfMappingLibrary } from "./models/ObservabilityPipelineOcsfMappingLibrary";
162167
export { ObservabilityPipelineOpenSearchDestination } from "./models/ObservabilityPipelineOpenSearchDestination";
163168
export { ObservabilityPipelineOpenSearchDestinationDataStream } from "./models/ObservabilityPipelineOpenSearchDestinationDataStream";

‎services/observability_pipelines/src/v2/models/ObservabilityPipelineOcsfMapperProcessorMappingMapping.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import { UnparsedObject } from "@datadog/datadog-api-client";
22

3+
import { ObservabilityPipelineOcsfMappingCustom } from "./ObservabilityPipelineOcsfMappingCustom";
34
import { ObservabilityPipelineOcsfMappingLibrary } from "./ObservabilityPipelineOcsfMappingLibrary";
45

56
/**
67
* Defines a single mapping rule for transforming logs into the OCSF schema.
78
*/
89
export type ObservabilityPipelineOcsfMapperProcessorMappingMapping =
910
| ObservabilityPipelineOcsfMappingLibrary
11+
| ObservabilityPipelineOcsfMappingCustom
1012
| UnparsedObject;

0 commit comments

Comments
 (0)