Skip to content

Commit 117a610

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add ObservabilityPipelineClickhouseDestination to OpenAPI spec (#1778)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent ef95994 commit 117a610

21 files changed

Lines changed: 1732 additions & 0 deletions

File tree

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63744,6 +63744,199 @@ components:
6374463744
x-enum-varnames:
6374563745
- BLOCK
6374663746
- DROP_NEWEST
63747+
ObservabilityPipelineClickhouseDestination:
63748+
description: |-
63749+
The `clickhouse` destination sends log events to a ClickHouse database table over HTTP.
63750+
63751+
**Supported pipeline types:** logs.
63752+
properties:
63753+
auth:
63754+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationAuth"
63755+
batch:
63756+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatch"
63757+
batch_encoding:
63758+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncoding"
63759+
compression:
63760+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompression"
63761+
database:
63762+
description: Optional ClickHouse database name. If omitted, the user's default database on the ClickHouse server is used.
63763+
example: my_database
63764+
type: string
63765+
date_time_best_effort:
63766+
description: When `true`, enables flexible DateTime parsing on the ClickHouse server side.
63767+
example: false
63768+
type: boolean
63769+
endpoint_url_key:
63770+
description: |-
63771+
Name of the environment variable or secret that contains the ClickHouse HTTP endpoint URL.
63772+
Defaults to `DESTINATION_CLICKHOUSE_ENDPOINT_URL` (prefixed with `DD_OP_` at runtime).
63773+
example: CLICKHOUSE_ENDPOINT_URL
63774+
type: string
63775+
format:
63776+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationFormat"
63777+
id:
63778+
description: The unique identifier for this component.
63779+
example: clickhouse-destination
63780+
type: string
63781+
inputs:
63782+
description: A list of component IDs whose output is used as the `input` for this component.
63783+
example: ["filter-processor"]
63784+
items:
63785+
description: The ID of a component whose output is used as input for this destination.
63786+
type: string
63787+
type: array
63788+
skip_unknown_fields:
63789+
description: |-
63790+
When `true`, fields not present in the target table schema are dropped instead of causing insert errors.
63791+
When unset, the ClickHouse server's own `input_format_skip_unknown_fields` setting applies.
63792+
example: true
63793+
nullable: true
63794+
type: boolean
63795+
table:
63796+
description: Target ClickHouse table name. Events are inserted into this table.
63797+
example: application_logs
63798+
type: string
63799+
tls:
63800+
$ref: "#/components/schemas/ObservabilityPipelineTls"
63801+
type:
63802+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationType"
63803+
required:
63804+
- id
63805+
- type
63806+
- inputs
63807+
- table
63808+
type: object
63809+
x-pipeline-types: [logs]
63810+
ObservabilityPipelineClickhouseDestinationAuth:
63811+
description: |-
63812+
HTTP Basic Authentication credentials for the ClickHouse destination.
63813+
When `strategy` is `basic`, provide `username_key` and `password_key` that reference environment variables or secrets containing the credentials.
63814+
properties:
63815+
password_key:
63816+
description: Name of the environment variable or secret that contains the ClickHouse password.
63817+
example: CLICKHOUSE_PASSWORD
63818+
type: string
63819+
strategy:
63820+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationAuthStrategy"
63821+
username_key:
63822+
description: Name of the environment variable or secret that contains the ClickHouse username.
63823+
example: CLICKHOUSE_USERNAME
63824+
type: string
63825+
required:
63826+
- strategy
63827+
type: object
63828+
ObservabilityPipelineClickhouseDestinationAuthStrategy:
63829+
description: The authentication strategy for ClickHouse HTTP requests. Only `basic` is supported.
63830+
enum:
63831+
- basic
63832+
example: basic
63833+
type: string
63834+
x-enum-varnames:
63835+
- BASIC
63836+
ObservabilityPipelineClickhouseDestinationBatch:
63837+
description: Batching configuration for ClickHouse inserts.
63838+
properties:
63839+
max_events:
63840+
description: Maximum number of events per batch before it is flushed.
63841+
example: 1000
63842+
format: int64
63843+
minimum: 1
63844+
type: integer
63845+
timeout_secs:
63846+
description: Maximum number of seconds to wait before flushing a partial batch.
63847+
example: 1
63848+
format: int64
63849+
maximum: 65535
63850+
minimum: 1
63851+
type: integer
63852+
type: object
63853+
ObservabilityPipelineClickhouseDestinationBatchEncoding:
63854+
description: |-
63855+
Batch encoding configuration for the ClickHouse destination.
63856+
Required when `format` is `arrow_stream`. The `codec` field must be set to `arrow_stream`.
63857+
properties:
63858+
allow_nullable_fields:
63859+
description: |-
63860+
When `true`, null values are allowed for non-nullable fields in the ClickHouse schema.
63861+
When `false` (default), missing values for non-nullable columns cause encoding errors.
63862+
example: false
63863+
type: boolean
63864+
codec:
63865+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationBatchEncodingCodec"
63866+
required:
63867+
- codec
63868+
type: object
63869+
ObservabilityPipelineClickhouseDestinationBatchEncodingCodec:
63870+
description: The codec used for batch encoding. Only `arrow_stream` is supported.
63871+
enum:
63872+
- arrow_stream
63873+
example: arrow_stream
63874+
type: string
63875+
x-enum-varnames:
63876+
- ARROW_STREAM
63877+
ObservabilityPipelineClickhouseDestinationCompression:
63878+
description: |-
63879+
Compression setting for outbound HTTP requests to ClickHouse.
63880+
Can be specified as a shorthand string (`"gzip"` or `"none"`) or as an object
63881+
with an `algorithm` field and an optional `level` (gzip only, 1–9).
63882+
oneOf:
63883+
- $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionAlgorithm"
63884+
- $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionObject"
63885+
ObservabilityPipelineClickhouseDestinationCompressionAlgorithm:
63886+
description: The compression algorithm applied to outbound HTTP requests.
63887+
enum:
63888+
- gzip
63889+
- none
63890+
example: gzip
63891+
type: string
63892+
x-enum-varnames:
63893+
- GZIP
63894+
- NONE
63895+
ObservabilityPipelineClickhouseDestinationCompressionObject:
63896+
description: |-
63897+
Structured compression configuration for the ClickHouse destination.
63898+
Use `algorithm` to specify the compression type and `level` (optional, gzip only) to control compression strength.
63899+
properties:
63900+
algorithm:
63901+
$ref: "#/components/schemas/ObservabilityPipelineClickhouseDestinationCompressionAlgorithm"
63902+
level:
63903+
description: Compression level (1–9). Only applicable when `algorithm` is `gzip`.
63904+
example: 6
63905+
format: int64
63906+
maximum: 9
63907+
minimum: 1
63908+
type: integer
63909+
required:
63910+
- algorithm
63911+
type: object
63912+
ObservabilityPipelineClickhouseDestinationFormat:
63913+
description: |-
63914+
Insert format for events sent to ClickHouse.
63915+
- `json_each_row`: Maps event fields to columns by name (ClickHouse `JSONEachRow`).
63916+
- `json_as_object`: Inserts each event into a single `Object('json')` / `JSON` column (ClickHouse `JSONAsObject`).
63917+
- `json_as_string`: Inserts each event into a single `String`-typed column as raw JSON (ClickHouse `JSONAsString`).
63918+
- `arrow_stream`: Batches events using Apache Arrow IPC streaming format. Requires `batch_encoding`.
63919+
enum:
63920+
- json_each_row
63921+
- json_as_object
63922+
- json_as_string
63923+
- arrow_stream
63924+
example: json_each_row
63925+
type: string
63926+
x-enum-varnames:
63927+
- JSON_EACH_ROW
63928+
- JSON_AS_OBJECT
63929+
- JSON_AS_STRING
63930+
- ARROW_STREAM
63931+
ObservabilityPipelineClickhouseDestinationType:
63932+
default: clickhouse
63933+
description: The destination type. The value must be `clickhouse`.
63934+
enum:
63935+
- clickhouse
63936+
example: clickhouse
63937+
type: string
63938+
x-enum-varnames:
63939+
- CLICKHOUSE
6374763940
ObservabilityPipelineCloudPremDestination:
6374863941
description: |-
6374963942
The `cloud_prem` destination sends logs to Datadog CloudPrem.
@@ -63843,6 +64036,7 @@ components:
6384364036
- $ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericDestination"
6384464037
- $ref: "#/components/schemas/ObservabilityPipelineAmazonSecurityLakeDestination"
6384564038
- $ref: "#/components/schemas/AzureStorageDestination"
64039+
- $ref: "#/components/schemas/ObservabilityPipelineClickhouseDestination"
6384664040
- $ref: "#/components/schemas/ObservabilityPipelineCloudPremDestination"
6384764041
- $ref: "#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestination"
6384864042
- $ref: "#/components/schemas/ObservabilityPipelineDatadogLogsDestination"

‎examples/v2_observability-pipelines_ValidatePipeline_1238441879.rs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Validate an observability pipeline with ClickHouse destination returns "OK"
2+
// response
3+
use datadog_api_client::datadog;
4+
use datadog_api_client::datadogV2::api_observability_pipelines::ObservabilityPipelinesAPI;
5+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestination;
6+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationAuth;
7+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationAuthStrategy;
8+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationBatch;
9+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationCompression;
10+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationCompressionAlgorithm;
11+
use datadog_api_client::datadogV2::model::ObservabilityPipelineClickhouseDestinationType;
12+
use datadog_api_client::datadogV2::model::ObservabilityPipelineConfig;
13+
use datadog_api_client::datadogV2::model::ObservabilityPipelineConfigDestinationItem;
14+
use datadog_api_client::datadogV2::model::ObservabilityPipelineConfigProcessorGroup;
15+
use datadog_api_client::datadogV2::model::ObservabilityPipelineConfigProcessorItem;
16+
use datadog_api_client::datadogV2::model::ObservabilityPipelineConfigSourceItem;
17+
use datadog_api_client::datadogV2::model::ObservabilityPipelineDataAttributes;
18+
use datadog_api_client::datadogV2::model::ObservabilityPipelineDatadogAgentSource;
19+
use datadog_api_client::datadogV2::model::ObservabilityPipelineDatadogAgentSourceType;
20+
use datadog_api_client::datadogV2::model::ObservabilityPipelineFilterProcessor;
21+
use datadog_api_client::datadogV2::model::ObservabilityPipelineFilterProcessorType;
22+
use datadog_api_client::datadogV2::model::ObservabilityPipelineSpec;
23+
use datadog_api_client::datadogV2::model::ObservabilityPipelineSpecData;
24+
25+
#[tokio::main]
26+
async fn main() {
27+
let body =
28+
ObservabilityPipelineSpec::new(
29+
ObservabilityPipelineSpecData::new(
30+
ObservabilityPipelineDataAttributes::new(
31+
ObservabilityPipelineConfig::new(
32+
vec![
33+
ObservabilityPipelineConfigDestinationItem::ObservabilityPipelineClickhouseDestination(
34+
Box::new(
35+
ObservabilityPipelineClickhouseDestination::new(
36+
"clickhouse-destination".to_string(),
37+
vec!["my-processor-group".to_string()],
38+
"application_logs".to_string(),
39+
ObservabilityPipelineClickhouseDestinationType::CLICKHOUSE,
40+
)
41+
.auth(
42+
ObservabilityPipelineClickhouseDestinationAuth::new(
43+
ObservabilityPipelineClickhouseDestinationAuthStrategy::BASIC,
44+
)
45+
.password_key("CLICKHOUSE_PASSWORD".to_string())
46+
.username_key("CLICKHOUSE_USERNAME".to_string()),
47+
)
48+
.batch(
49+
ObservabilityPipelineClickhouseDestinationBatch::new()
50+
.max_events(1000)
51+
.timeout_secs(1),
52+
)
53+
.compression(
54+
ObservabilityPipelineClickhouseDestinationCompression
55+
::ObservabilityPipelineClickhouseDestinationCompressionAlgorithm(
56+
Box::new(
57+
ObservabilityPipelineClickhouseDestinationCompressionAlgorithm
58+
::GZIP,
59+
),
60+
),
61+
)
62+
.database("my_database".to_string()),
63+
),
64+
)
65+
],
66+
vec![
67+
ObservabilityPipelineConfigSourceItem::ObservabilityPipelineDatadogAgentSource(
68+
Box::new(
69+
ObservabilityPipelineDatadogAgentSource::new(
70+
"datadog-agent-source".to_string(),
71+
ObservabilityPipelineDatadogAgentSourceType::DATADOG_AGENT,
72+
),
73+
),
74+
)
75+
],
76+
).processor_groups(
77+
vec![
78+
ObservabilityPipelineConfigProcessorGroup::new(
79+
true,
80+
"my-processor-group".to_string(),
81+
"service:my-service".to_string(),
82+
vec!["datadog-agent-source".to_string()],
83+
vec![
84+
ObservabilityPipelineConfigProcessorItem::ObservabilityPipelineFilterProcessor(
85+
Box::new(
86+
ObservabilityPipelineFilterProcessor::new(
87+
true,
88+
"filter-processor".to_string(),
89+
"status:error".to_string(),
90+
ObservabilityPipelineFilterProcessorType::FILTER,
91+
),
92+
),
93+
)
94+
],
95+
)
96+
],
97+
),
98+
"Pipeline with ClickHouse Destination".to_string(),
99+
),
100+
"pipelines".to_string(),
101+
),
102+
);
103+
let configuration = datadog::Configuration::new();
104+
let api = ObservabilityPipelinesAPI::with_config(configuration);
105+
let resp = api.validate_pipeline(body).await;
106+
if let Ok(value) = resp {
107+
println!("{:#?}", value);
108+
} else {
109+
println!("{:#?}", resp.unwrap_err());
110+
}
111+
}

0 commit comments

Comments
 (0)