Skip to content

Commit 55fa3e4

Browse files
shuheiktgwshuheiktgw
committed
Split telemetry exporters by backend
1 parent c8f762b commit 55fa3e4

11 files changed

Lines changed: 259 additions & 177 deletions

File tree

‎quickwit/quickwit-serve/src/metrics_api.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ pub struct MetricsApi;
3838
///
3939
/// These are in the form of prometheus metrics.
4040
pub fn metrics_handler() -> impl warp::Reply {
41-
match quickwit_telemetry_exporters::metrics::metrics_text_payload() {
41+
match quickwit_telemetry_exporters::prometheus::metrics::text_payload() {
4242
Ok(metrics) => with_status(metrics, StatusCode::OK),
4343
Err(e) => {
4444
error!("failed to encode prometheus metrics: {e}");

‎quickwit/quickwit-telemetry-exporters/src/lib.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ use tracing_subscriber::EnvFilter;
3131
use tracing_subscriber::layer::SubscriberExt;
3232
use tracing_subscriber::prelude::*;
3333

34-
mod config;
3534
mod logs;
36-
pub mod metrics;
37-
mod trace;
35+
mod metrics;
36+
mod otlp;
37+
pub mod prometheus;
3838

3939
#[cfg(feature = "tokio-console")]
4040
const QW_ENABLE_TOKIO_CONSOLE_ENV_KEY: &str = "QW_ENABLE_TOKIO_CONSOLE";
@@ -95,7 +95,7 @@ pub fn init_telemetry(
9595
level: Level,
9696
ansi_colors: bool,
9797
) -> anyhow::Result<TelemetryHandle> {
98-
let otlp_config = config::OtlpExporterConfig::load_from_env();
98+
let otlp_config = otlp::OtlpExporterConfig::load_from_env();
9999

100100
let meter_provider = metrics::init_metrics_provider(service_version, &otlp_config)?;
101101

@@ -146,10 +146,10 @@ pub fn init_telemetry(
146146
// Note on disabling ANSI characters: setting the ansi boolean on event format is insufficient.
147147
// It is thus set on layers, see https://github.com/tokio-rs/tracing/issues/1817
148148
let telemetry_handle = if otlp_config.is_enabled() {
149-
let resource = config::quickwit_resource(service_version);
149+
let resource = otlp::quickwit_resource(service_version);
150150

151-
let tracer_provider = trace::init_tracer_provider(&otlp_config, resource.clone())?;
152-
let logger_provider = logs::init_logger_provider(&otlp_config, resource)?;
151+
let tracer_provider = otlp::traces::init_tracer_provider(&otlp_config, resource.clone())?;
152+
let logger_provider = otlp::logs::init_logger_provider(&otlp_config, resource)?;
153153

154154
let tracer = tracer_provider.tracer("quickwit");
155155
let telemetry_layer = tracing_opentelemetry::layer().with_tracer(tracer);

‎quickwit/quickwit-telemetry-exporters/src/logs.rs‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,6 @@
1414

1515
use std::fmt;
1616

17-
use anyhow::Context;
18-
use opentelemetry_otlp::{LogExporter, Protocol as OtlpWireProtocol, WithExportConfig};
19-
use opentelemetry_sdk::Resource;
20-
use opentelemetry_sdk::logs::SdkLoggerProvider;
2117
use quickwit_common::get_from_env_opt;
2218
use time::format_description::BorrowedFormatItem;
2319
use tracing::{Event, Subscriber};
@@ -29,37 +25,6 @@ use tracing_subscriber::fmt::format::{
2925
use tracing_subscriber::fmt::time::UtcTime;
3026
use tracing_subscriber::registry::LookupSpan;
3127

32-
use crate::config::{OtlpExporterConfig, OtlpProtocol};
33-
34-
impl OtlpProtocol {
35-
pub(crate) fn log_exporter(&self) -> anyhow::Result<LogExporter> {
36-
match self {
37-
OtlpProtocol::Grpc => LogExporter::builder().with_tonic().build(),
38-
OtlpProtocol::HttpProtobuf => LogExporter::builder()
39-
.with_http()
40-
.with_protocol(OtlpWireProtocol::HttpBinary)
41-
.build(),
42-
OtlpProtocol::HttpJson => LogExporter::builder()
43-
.with_http()
44-
.with_protocol(OtlpWireProtocol::HttpJson)
45-
.build(),
46-
}
47-
.context("failed to initialize OTLP logs exporter")
48-
}
49-
}
50-
51-
pub(crate) fn init_logger_provider(
52-
otlp_config: &OtlpExporterConfig,
53-
resource: Resource,
54-
) -> anyhow::Result<SdkLoggerProvider> {
55-
let logs_protocol = otlp_config.logs_protocol()?;
56-
let log_exporter = logs_protocol.log_exporter()?;
57-
Ok(SdkLoggerProvider::builder()
58-
.with_resource(resource)
59-
.with_batch_exporter(log_exporter)
60-
.build())
61-
}
62-
6328
/// We do not rely on the RFC3339 implementation, because it has a nanosecond precision.
6429
/// See discussion here: https://github.com/time-rs/time/discussions/418
6530
pub(crate) fn time_formatter() -> UtcTime<Vec<BorrowedFormatItem<'static>>> {

‎quickwit/quickwit-telemetry-exporters/src/metrics.rs‎

Lines changed: 5 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -12,57 +12,21 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
use std::sync::OnceLock;
16-
use std::time::Duration;
17-
18-
use anyhow::Context;
19-
use metrics_exporter_otel::OpenTelemetryRecorder;
20-
use metrics_exporter_prometheus::{
21-
Matcher, PrometheusBuilder, PrometheusHandle, PrometheusRecorder,
22-
};
2315
use metrics_util::layers::FanoutBuilder;
24-
use opentelemetry::metrics::MeterProvider;
25-
use opentelemetry_otlp::{MetricExporter, Protocol as OtlpWireProtocol, WithExportConfig};
26-
use opentelemetry_sdk::metrics::{SdkMeterProvider, Temporality};
27-
28-
use crate::config::{OtlpExporterConfig, OtlpProtocol, quickwit_resource};
29-
30-
static PROMETHEUS_HANDLE: OnceLock<PrometheusHandle> = OnceLock::new();
16+
use opentelemetry_sdk::metrics::SdkMeterProvider;
3117

32-
impl OtlpProtocol {
33-
pub(crate) fn metric_exporter(
34-
&self,
35-
temporality: Temporality,
36-
) -> anyhow::Result<MetricExporter> {
37-
match self {
38-
OtlpProtocol::Grpc => MetricExporter::builder()
39-
.with_tonic()
40-
.with_temporality(temporality)
41-
.build(),
42-
OtlpProtocol::HttpProtobuf => MetricExporter::builder()
43-
.with_http()
44-
.with_temporality(temporality)
45-
.with_protocol(OtlpWireProtocol::HttpBinary)
46-
.build(),
47-
OtlpProtocol::HttpJson => MetricExporter::builder()
48-
.with_http()
49-
.with_temporality(temporality)
50-
.with_protocol(OtlpWireProtocol::HttpJson)
51-
.build(),
52-
}
53-
.context("failed to initialize OTLP metrics exporter")
54-
}
55-
}
18+
use crate::otlp::OtlpExporterConfig;
5619

5720
/// Sets up the global metrics recorder.
5821
pub(crate) fn init_metrics_provider(
5922
service_version: &str,
6023
otlp_config: &OtlpExporterConfig,
6124
) -> anyhow::Result<Option<SdkMeterProvider>> {
62-
let prometheus_recorder = build_prometheus_recorder()?;
25+
let prometheus_recorder = crate::prometheus::metrics::build_recorder()?;
6326

6427
let (recorder, meter_provider) = if otlp_config.is_enabled() {
65-
let (otlp_recorder, meter_provider) = build_otlp_recorder(service_version, otlp_config)?;
28+
let (otlp_recorder, meter_provider) =
29+
crate::otlp::metrics::build_recorder(service_version, otlp_config)?;
6630
let recorder = FanoutBuilder::default()
6731
.add_recorder(prometheus_recorder)
6832
.add_recorder(otlp_recorder)
@@ -81,95 +45,3 @@ pub(crate) fn init_metrics_provider(
8145

8246
Ok(meter_provider)
8347
}
84-
85-
fn build_prometheus_recorder() -> anyhow::Result<PrometheusRecorder> {
86-
let mut prometheus_builder = PrometheusBuilder::new();
87-
for (name, buckets) in quickwit_metrics::histogram_buckets() {
88-
prometheus_builder = prometheus_builder
89-
.set_buckets_for_metric(Matcher::Full(name.to_string()), &buckets)
90-
.with_context(|| {
91-
format!("failed to configure Prometheus histogram buckets for `{name}`")
92-
})?;
93-
}
94-
let prometheus_recorder = prometheus_builder.build_recorder();
95-
let prometheus_handle = prometheus_recorder.handle();
96-
PROMETHEUS_HANDLE
97-
.set(prometheus_handle.clone())
98-
.map_err(|_| anyhow::anyhow!("Prometheus metrics renderer is already installed"))?;
99-
spawn_prometheus_upkeep(prometheus_handle).map_err(anyhow::Error::msg)?;
100-
Ok(prometheus_recorder)
101-
}
102-
103-
pub fn metrics_text_payload() -> Result<String, String> {
104-
let handle = PROMETHEUS_HANDLE
105-
.get()
106-
.ok_or_else(|| "Prometheus metrics rendering is not installed yet".to_string())?;
107-
Ok(handle.render())
108-
}
109-
110-
fn spawn_prometheus_upkeep(handle: PrometheusHandle) -> Result<(), String> {
111-
// Quickwit serves the existing `/metrics` route itself, so we build only the
112-
// Prometheus recorder instead of using the exporter's HTTP listener. That lower-level
113-
// API does not spawn the upkeep task that periodically drains histogram buffers.
114-
std::thread::Builder::new()
115-
.name("telemetry-exporter-prometheus-upkeep".to_string())
116-
.spawn(move || {
117-
loop {
118-
std::thread::sleep(Duration::from_secs(5));
119-
handle.run_upkeep();
120-
}
121-
})
122-
.map(|_| ())
123-
.map_err(|error| format!("failed to spawn Prometheus metrics upkeep thread: {error}"))
124-
}
125-
126-
fn build_otlp_recorder(
127-
service_version: &str,
128-
otlp_config: &OtlpExporterConfig,
129-
) -> anyhow::Result<(OpenTelemetryRecorder, SdkMeterProvider)> {
130-
let metrics_protocol = otlp_config.metrics_protocol()?;
131-
let temporality = otlp_config.metrics_temporality()?;
132-
let metric_exporter = metrics_protocol.metric_exporter(temporality)?;
133-
let metrics_provider = SdkMeterProvider::builder()
134-
.with_resource(quickwit_resource(service_version))
135-
.with_periodic_exporter(metric_exporter)
136-
.build();
137-
let meter = metrics_provider.meter("quickwit");
138-
139-
let recorder = OpenTelemetryRecorder::new(meter);
140-
for (name, buckets) in quickwit_metrics::histogram_buckets() {
141-
recorder.set_histogram_bounds(&metrics::KeyName::from(name), buckets);
142-
}
143-
Ok((recorder, metrics_provider))
144-
}
145-
146-
#[cfg(test)]
147-
mod tests {
148-
use metrics::with_local_recorder;
149-
use metrics_exporter_prometheus::PrometheusBuilder;
150-
use quickwit_metrics::{gauge, labels};
151-
152-
use super::*;
153-
154-
#[test]
155-
fn metrics_text_payload_renders_prometheus_handle() {
156-
let recorder = PrometheusBuilder::new().build_recorder();
157-
PROMETHEUS_HANDLE
158-
.set(recorder.handle())
159-
.expect("Prometheus handle should be set once");
160-
161-
with_local_recorder(&recorder, || {
162-
let info_metric = gauge!(
163-
name: "prometheus_payload_info",
164-
description: "prometheus payload info",
165-
subsystem: "",
166-
);
167-
quickwit_metrics::describe_metrics();
168-
gauge!(parent: info_metric, labels: [labels!("version" => "test")]).set(1.0);
169-
});
170-
171-
let payload = metrics_text_payload().expect("Prometheus payload should render");
172-
assert!(payload.contains("# HELP quickwit_prometheus_payload_info"));
173-
assert!(payload.contains(r#"quickwit_prometheus_payload_info{version="test"} 1"#));
174-
}
175-
}

quickwit/quickwit-telemetry-exporters/src/config.rs renamed to quickwit/quickwit-telemetry-exporters/src/otlp/config.rs

File renamed without changes.

‎quickwit/quickwit-telemetry-exporters/src/otlp/logs.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright 2021-Present Datadog, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use anyhow::Context;
16+
use opentelemetry_otlp::{LogExporter, Protocol as OtlpWireProtocol, WithExportConfig};
17+
use opentelemetry_sdk::Resource;
18+
use opentelemetry_sdk::logs::SdkLoggerProvider;
19+
20+
use crate::otlp::{OtlpExporterConfig, OtlpProtocol};
21+
22+
impl OtlpProtocol {
23+
pub(crate) fn log_exporter(&self) -> anyhow::Result<LogExporter> {
24+
match self {
25+
OtlpProtocol::Grpc => LogExporter::builder().with_tonic().build(),
26+
OtlpProtocol::HttpProtobuf => LogExporter::builder()
27+
.with_http()
28+
.with_protocol(OtlpWireProtocol::HttpBinary)
29+
.build(),
30+
OtlpProtocol::HttpJson => LogExporter::builder()
31+
.with_http()
32+
.with_protocol(OtlpWireProtocol::HttpJson)
33+
.build(),
34+
}
35+
.context("failed to initialize OTLP logs exporter")
36+
}
37+
}
38+
39+
pub(crate) fn init_logger_provider(
40+
otlp_config: &OtlpExporterConfig,
41+
resource: Resource,
42+
) -> anyhow::Result<SdkLoggerProvider> {
43+
let logs_protocol = otlp_config.logs_protocol()?;
44+
let log_exporter = logs_protocol.log_exporter()?;
45+
Ok(SdkLoggerProvider::builder()
46+
.with_resource(resource)
47+
.with_batch_exporter(log_exporter)
48+
.build())
49+
}

‎quickwit/quickwit-telemetry-exporters/src/otlp/metrics.rs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright 2021-Present Datadog, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
use anyhow::Context;
16+
use metrics_exporter_otel::OpenTelemetryRecorder;
17+
use opentelemetry::metrics::MeterProvider;
18+
use opentelemetry_otlp::{MetricExporter, Protocol as OtlpWireProtocol, WithExportConfig};
19+
use opentelemetry_sdk::metrics::{SdkMeterProvider, Temporality};
20+
21+
use crate::otlp::{OtlpExporterConfig, OtlpProtocol, quickwit_resource};
22+
23+
impl OtlpProtocol {
24+
pub(crate) fn metric_exporter(
25+
&self,
26+
temporality: Temporality,
27+
) -> anyhow::Result<MetricExporter> {
28+
match self {
29+
OtlpProtocol::Grpc => MetricExporter::builder()
30+
.with_tonic()
31+
.with_temporality(temporality)
32+
.build(),
33+
OtlpProtocol::HttpProtobuf => MetricExporter::builder()
34+
.with_http()
35+
.with_temporality(temporality)
36+
.with_protocol(OtlpWireProtocol::HttpBinary)
37+
.build(),
38+
OtlpProtocol::HttpJson => MetricExporter::builder()
39+
.with_http()
40+
.with_temporality(temporality)
41+
.with_protocol(OtlpWireProtocol::HttpJson)
42+
.build(),
43+
}
44+
.context("failed to initialize OTLP metrics exporter")
45+
}
46+
}
47+
48+
pub(crate) fn build_recorder(
49+
service_version: &str,
50+
otlp_config: &OtlpExporterConfig,
51+
) -> anyhow::Result<(OpenTelemetryRecorder, SdkMeterProvider)> {
52+
let metrics_protocol = otlp_config.metrics_protocol()?;
53+
let temporality = otlp_config.metrics_temporality()?;
54+
let metric_exporter = metrics_protocol.metric_exporter(temporality)?;
55+
let metrics_provider = SdkMeterProvider::builder()
56+
.with_resource(quickwit_resource(service_version))
57+
.with_periodic_exporter(metric_exporter)
58+
.build();
59+
let meter = metrics_provider.meter("quickwit");
60+
61+
let recorder = OpenTelemetryRecorder::new(meter);
62+
for (name, buckets) in quickwit_metrics::histogram_buckets() {
63+
recorder.set_histogram_bounds(&metrics::KeyName::from(name), buckets);
64+
}
65+
Ok((recorder, metrics_provider))
66+
}

‎quickwit/quickwit-telemetry-exporters/src/otlp/mod.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2021-Present Datadog, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
mod config;
16+
pub(crate) mod logs;
17+
pub(crate) mod metrics;
18+
pub(crate) mod traces;
19+
20+
pub(crate) use config::{OtlpExporterConfig, OtlpProtocol, quickwit_resource};

0 commit comments

Comments
 (0)