Skip to content

Commit 1b4b37f

Browse files
bwoebibwoebi
committed
Use a sidecar connection per PHP thread
While the sidecar has been a process global for the longest time, it's also unfair with respect to limits and not compatible with the newly introduced outbox. Having a connection per thread ensures that every thread can have up to 100 queued items. It also reduces contention on the SidecarTransport, which was indeed adding quite a bit of latency due to threads accumulating in futex_wait for very short requests under moderate load. Moving the background sender stats to its own thread in preparation of dropping the Mutex on the SidecarTransport fully. Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>
1 parent 8ecdabc commit 1b4b37f

File tree

13 files changed

+276
-208
lines changed

13 files changed

+276
-208
lines changed

‎components-rs/ddtrace.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@ extern ddog_VecRemoteConfigCapabilities DDTRACE_REMOTE_CONFIG_CAPABILITIES;
1919

2020
extern const uint8_t *DDOG_PHP_FUNCTION;
2121

22-
extern struct ddog_SidecarTransport *ddtrace_sidecar;
23-
2422
/**
2523
* # Safety
2624
* Must be called from a single-threaded context, such as MINIT.

‎components-rs/sidecar.rs‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,10 +170,6 @@ pub extern "C" fn ddog_sidecar_connect_php(
170170
MaybeError::None
171171
}
172172

173-
#[no_mangle]
174-
#[allow(non_upper_case_globals)]
175-
pub static mut ddtrace_sidecar: *mut SidecarTransport = std::ptr::null_mut();
176-
177173
#[no_mangle]
178174
pub extern "C" fn ddtrace_sidecar_reconnect(
179175
transport: &mut Box<SidecarTransport>,

‎ext/auto_flush.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ ZEND_RESULT_CODE ddtrace_flush_tracer(bool force_on_startup, bool collect_cycles
4747
char *url = ddtrace_agent_url();
4848

4949
if (get_global_DD_TRACE_SIDECAR_TRACE_SENDER()) {
50-
if (ddtrace_sidecar) {
50+
if (DDTRACE_G(sidecar)) {
5151
ddog_SenderParameters parameters = {
5252
.tracer_headers_tags = {
5353
.container_id = ddtrace_get_container_id(),
@@ -59,7 +59,7 @@ ZEND_RESULT_CODE ddtrace_flush_tracer(bool force_on_startup, bool collect_cycles
5959
.client_computed_top_level = false,
6060
.client_computed_stats = !get_global_DD_APM_TRACING_ENABLED(),
6161
},
62-
.transport = ddtrace_sidecar,
62+
.transport = DDTRACE_G(sidecar),
6363
.instance_id = ddtrace_sidecar_instance_id,
6464
.limit = limit,
6565
.n_requests = get_global_DD_TRACE_AGENT_FLUSH_AFTER_N_REQUESTS(),

‎ext/ddtrace.c‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -409,21 +409,21 @@ static inline void dd_alter_prop(size_t prop_offset, zval *old_value, zval *new_
409409
bool ddtrace_alter_dd_service(zval *old_value, zval *new_value, zend_string *new_str) {
410410
dd_alter_prop(XtOffsetOf(ddtrace_span_properties, property_service), old_value, new_value, new_str);
411411
if (DDTRACE_G(request_initialized)) {
412-
ddtrace_sidecar_submit_root_span_data_direct(&ddtrace_sidecar, NULL, new_str, get_DD_ENV(), get_DD_VERSION());
412+
ddtrace_sidecar_submit_root_span_data_direct(&DDTRACE_G(sidecar), NULL, new_str, get_DD_ENV(), get_DD_VERSION());
413413
}
414414
return true;
415415
}
416416
bool ddtrace_alter_dd_env(zval *old_value, zval *new_value, zend_string *new_str) {
417417
dd_alter_prop(XtOffsetOf(ddtrace_span_properties, property_env), old_value, new_value, new_str);
418418
if (DDTRACE_G(request_initialized)) {
419-
ddtrace_sidecar_submit_root_span_data_direct(&ddtrace_sidecar, NULL, get_DD_SERVICE(), new_str, get_DD_VERSION());
419+
ddtrace_sidecar_submit_root_span_data_direct(&DDTRACE_G(sidecar), NULL, get_DD_SERVICE(), new_str, get_DD_VERSION());
420420
}
421421
return true;
422422
}
423423
bool ddtrace_alter_dd_version(zval *old_value, zval *new_value, zend_string *new_str) {
424424
dd_alter_prop(XtOffsetOf(ddtrace_span_properties, property_version), old_value, new_value, new_str);
425425
if (DDTRACE_G(request_initialized)) {
426-
ddtrace_sidecar_submit_root_span_data_direct(&ddtrace_sidecar, NULL, get_DD_SERVICE(), get_DD_ENV(), new_str);
426+
ddtrace_sidecar_submit_root_span_data_direct(&DDTRACE_G(sidecar), NULL, get_DD_SERVICE(), get_DD_ENV(), new_str);
427427
}
428428
return true;
429429
}
@@ -697,6 +697,9 @@ static PHP_GSHUTDOWN_FUNCTION(ddtrace) {
697697

698698
zend_hash_destroy(&ddtrace_globals->git_metadata);
699699

700+
// Drop the per-thread sidecar transport (thread-lifetime, one per thread).
701+
ddtrace_sidecar_gshutdown();
702+
700703
tsrm_mutex_free(ddtrace_globals->sidecar_universal_service_tags_mutex);
701704

702705
#ifdef CXA_THREAD_ATEXIT_WRAPPER
@@ -1594,8 +1597,8 @@ static PHP_MSHUTDOWN_FUNCTION(ddtrace) {
15941597
ddtrace_coms_mshutdown_proxy_env();
15951598
} else /* ! part of the if outside the ifdef */
15961599
#endif
1597-
if (get_global_DD_TRACE_FORCE_FLUSH_ON_SHUTDOWN() && ddtrace_sidecar) {
1598-
ddog_sidecar_flush_traces(&ddtrace_sidecar);
1600+
if (get_global_DD_TRACE_FORCE_FLUSH_ON_SHUTDOWN() && DDTRACE_G(sidecar)) {
1601+
ddog_sidecar_flush_traces(&DDTRACE_G(sidecar));
15991602
}
16001603

16011604
ddtrace_log_mshutdown();
@@ -2811,7 +2814,7 @@ PHP_FUNCTION(DDTrace_dogstatsd_set) {
28112814
PHP_FUNCTION(DDTrace_are_endpoints_collected) {
28122815
UNUSED(execute_data);
28132816

2814-
if (!ddtrace_sidecar || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id)) {
2817+
if (!DDTRACE_G(sidecar) || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id)) {
28152818
RETURN_TRUE; // Skip overhead if unnecessary
28162819
}
28172820

@@ -2867,7 +2870,7 @@ PHP_FUNCTION(DDTrace_add_endpoint) {
28672870
RETURN_FALSE;
28682871
}
28692872

2870-
if (!ddtrace_sidecar || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id)) {
2873+
if (!DDTRACE_G(sidecar) || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id)) {
28712874
RETURN_FALSE;
28722875
}
28732876

@@ -2892,7 +2895,7 @@ PHP_FUNCTION(DDTrace_flush_endpoints) {
28922895
UNUSED(execute_data);
28932896
UNUSED(return_value);
28942897

2895-
if (!ddtrace_sidecar || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id) || !DDTRACE_G(telemetry_buffer)) {
2898+
if (!DDTRACE_G(sidecar) || !ddtrace_sidecar_instance_id || !DDTRACE_G(sidecar_queue_id) || !DDTRACE_G(telemetry_buffer)) {
28962899
return;
28972900
}
28982901

@@ -2904,7 +2907,7 @@ PHP_FUNCTION(DDTrace_flush_endpoints) {
29042907
ddog_CharSlice env_name = dd_zend_string_to_CharSlice(DDTRACE_G(last_env_name));
29052908

29062909
ddtrace_ffi_try("Failed flushing endpoint telemetry buffer",
2907-
ddog_sidecar_telemetry_filter_flush(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), ddtrace_telemetry_buffer(), ddtrace_telemetry_cache(), service_name, env_name));
2910+
ddog_sidecar_telemetry_filter_flush(&DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), ddtrace_telemetry_buffer(), ddtrace_telemetry_cache(), service_name, env_name));
29082911
}
29092912

29102913
PHP_FUNCTION(dd_trace_send_traces_via_thread) {
@@ -3006,7 +3009,7 @@ PHP_FUNCTION(dd_trace_internal_fn) {
30063009
}
30073010
} else if (params_count == 1 && FUNCTION_NAME_MATCHES("detect_composer_installed_json")) {
30083011
ddog_CharSlice path = dd_zend_string_to_CharSlice(Z_STR_P(ZVAL_VARARG_PARAM(params, 0)));
3009-
ddtrace_detect_composer_installed_json(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), path);
3012+
ddtrace_detect_composer_installed_json(&DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), path);
30103013
RETVAL_TRUE;
30113014
} else if (params_count == 2 && FUNCTION_NAME_MATCHES("mark_integration_loaded")) {
30123015
zval *name = ZVAL_VARARG_PARAM(params, 0);
@@ -3034,24 +3037,24 @@ PHP_FUNCTION(dd_trace_internal_fn) {
30343037
RETVAL_TRUE;
30353038
}
30363039
} else if (FUNCTION_NAME_MATCHES("dump_sidecar")) {
3037-
if (!ddtrace_sidecar) {
3040+
if (!DDTRACE_G(sidecar)) {
30383041
RETURN_FALSE;
30393042
}
3040-
ddog_CharSlice slice = ddog_sidecar_dump(&ddtrace_sidecar);
3043+
ddog_CharSlice slice = ddog_sidecar_dump(&DDTRACE_G(sidecar));
30413044
RETVAL_STRINGL(slice.ptr, slice.len);
30423045
free((void *) slice.ptr);
30433046
} else if (FUNCTION_NAME_MATCHES("stats_sidecar")) {
3044-
if (!ddtrace_sidecar) {
3047+
if (!DDTRACE_G(sidecar)) {
30453048
RETURN_FALSE;
30463049
}
3047-
ddog_CharSlice slice = ddog_sidecar_stats(&ddtrace_sidecar);
3050+
ddog_CharSlice slice = ddog_sidecar_stats(&DDTRACE_G(sidecar));
30483051
RETVAL_STRINGL(slice.ptr, slice.len);
30493052
free((void *) slice.ptr);
30503053
} else if (FUNCTION_NAME_MATCHES("break_sidecar_connection")) {
3051-
if (!ddtrace_sidecar) {
3054+
if (!DDTRACE_G(sidecar)) {
30523055
RETURN_FALSE;
30533056
}
3054-
ddog_sidecar_send_garbage(&ddtrace_sidecar);
3057+
ddog_sidecar_send_garbage(&DDTRACE_G(sidecar));
30553058
ddtrace_generate_runtime_id();
30563059
ddtrace_force_new_instance_id();
30573060
RETURN_TRUE;
@@ -3081,8 +3084,8 @@ PHP_FUNCTION(dd_trace_internal_fn) {
30813084
}
30823085
} else
30833086
#endif
3084-
if (ddtrace_sidecar) {
3085-
ddtrace_ffi_try("Failed synchronously flushing traces", ddog_sidecar_flush_traces(&ddtrace_sidecar));
3087+
if (DDTRACE_G(sidecar)) {
3088+
ddtrace_ffi_try("Failed synchronously flushing traces", ddog_sidecar_flush_traces(&DDTRACE_G(sidecar)));
30863089
}
30873090
RETVAL_TRUE;
30883091
#ifndef _WIN32
@@ -3198,8 +3201,8 @@ PHP_FUNCTION(dd_trace_synchronous_flush) {
31983201
}
31993202
} else
32003203
#endif
3201-
if (ddtrace_sidecar) {
3202-
ddtrace_ffi_try("Failed synchronously flushing traces", ddog_sidecar_flush_traces(&ddtrace_sidecar));
3204+
if (DDTRACE_G(sidecar)) {
3205+
ddtrace_ffi_try("Failed synchronously flushing traces", ddog_sidecar_flush_traces(&DDTRACE_G(sidecar)));
32033206
}
32043207
RETURN_NULL();
32053208
}

‎ext/ddtrace.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,7 @@ ZEND_BEGIN_MODULE_GLOBALS(ddtrace)
149149
zend_reference *curl_multi_injecting_spans;
150150

151151
char *cgroup_file;
152+
ddog_SidecarTransport *sidecar;
152153
ddog_QueueId sidecar_queue_id;
153154
MUTEX_T sidecar_universal_service_tags_mutex;
154155
ddog_AgentRemoteConfigReader *agent_config_reader;
@@ -166,6 +167,7 @@ ZEND_BEGIN_MODULE_GLOBALS(ddtrace)
166167
bool request_initialized;
167168
HashTable telemetry_spans_created_per_integration;
168169
ddog_SidecarActionsBuffer *telemetry_buffer;
170+
ddog_SidecarActionsBuffer *metrics_buffer;
169171

170172
bool asm_event_emitted;
171173

‎ext/exception_serialize.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_st
384384
ddog_add_str_span_meta_str(span, "error.debug_info_captured", "true");
385385
ddog_add_str_span_meta_CharSlice(span, "_dd.debug.error.exception_hash", (ddog_CharSlice){.ptr = exception_hash, .len = hash_len});
386386

387-
if (!ddog_exception_hash_limiter_inc(ddtrace_sidecar, (uint64_t)exception_long_hash, get_DD_EXCEPTION_REPLAY_CAPTURE_INTERVAL_SECONDS())) {
387+
if (!ddog_exception_hash_limiter_inc(DDTRACE_G(sidecar), (uint64_t)exception_long_hash, get_DD_EXCEPTION_REPLAY_CAPTURE_INTERVAL_SECONDS())) {
388388
LOG(TRACE, "Skipping exception replay capture due to hash %.*s already recently hit", hash_len, exception_hash);
389389
goto cleanup;
390390
}

‎ext/live_debugger.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ static void dd_probe_resolved(void *data, bool found) {
151151
def->probe.status_msg = DDOG_CHARSLICE_C("Method does not exist on the given class");
152152
def->probe.status_exception = DDOG_CHARSLICE_C("METHOD_NOT_FOUND");
153153
}
154-
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &ddtrace_sidecar, ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
154+
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
155155
}
156156

157157
static int64_t dd_init_live_debugger_probe(const ddog_Probe *probe, dd_probe_def *def, zai_hook_begin begin, zai_hook_end end, void (*def_dtor)(void *), size_t dynamic) {
@@ -202,14 +202,14 @@ static int64_t dd_init_live_debugger_probe(const ddog_Probe *probe, dd_probe_def
202202
def->probe.status_msg = DDOG_CHARSLICE_C("Method does not exist on the given class");
203203
def->probe.status_exception = DDOG_CHARSLICE_C("METHOD_NOT_FOUND");
204204
error:
205-
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &ddtrace_sidecar, ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
205+
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
206206
def_dtor(def);
207207
return -1;
208208
}
209209

210210
if (def->probe.status != DDOG_PROBE_STATUS_INSTALLED) {
211211
def->probe.status = DDOG_PROBE_STATUS_RECEIVED;
212-
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &ddtrace_sidecar, ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
212+
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
213213
}
214214

215215
zend_hash_index_add_new_ptr(&DDTRACE_G(active_rc_hooks), id, def);
@@ -219,7 +219,7 @@ static int64_t dd_init_live_debugger_probe(const ddog_Probe *probe, dd_probe_def
219219
static void dd_probe_mark_active(dd_probe_def *def) {
220220
if (def->probe.status != DDOG_PROBE_STATUS_EMITTING) {
221221
def->probe.status = DDOG_PROBE_STATUS_EMITTING;
222-
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &ddtrace_sidecar, ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
222+
ddog_send_debugger_diagnostics(DDTRACE_G(remote_config_state), &DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, DDTRACE_G(sidecar_queue_id), &def->probe, ddtrace_nanoseconds_realtime() / 1000000);
223223
}
224224
}
225225

@@ -1468,8 +1468,8 @@ bool ddtrace_alter_dynamic_instrumentation_config(zval *old_value, zval *new_val
14681468
return false;
14691469
}
14701470

1471-
if (DDTRACE_G(request_initialized) && ddtrace_sidecar) {
1472-
ddog_sidecar_set_request_config(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), Z_TYPE_P(new_value) == IS_TRUE ? DDOG_DYNAMIC_INSTRUMENTATION_CONFIG_STATE_ENABLED : DDOG_DYNAMIC_INSTRUMENTATION_CONFIG_STATE_DISABLED);
1471+
if (DDTRACE_G(request_initialized) && DDTRACE_G(sidecar)) {
1472+
ddog_sidecar_set_request_config(&DDTRACE_G(sidecar), ddtrace_sidecar_instance_id, &DDTRACE_G(sidecar_queue_id), Z_TYPE_P(new_value) == IS_TRUE ? DDOG_DYNAMIC_INSTRUMENTATION_CONFIG_STATE_ENABLED : DDOG_DYNAMIC_INSTRUMENTATION_CONFIG_STATE_DISABLED);
14731473
}
14741474
return true;
14751475
}

‎ext/otel_config.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
ZEND_EXTERN_MODULE_GLOBALS(ddtrace);
1010

1111
static void report_otel_cfg_telemetry_invalid(const char *otel_cfg, const char *dd_cfg, bool pre_rinit) {
12-
if (!pre_rinit && ddtrace_sidecar && get_DD_INSTRUMENTATION_TELEMETRY_ENABLED()) {
13-
ddog_sidecar_telemetry_register_metric(&ddtrace_sidecar, DDOG_CHARSLICE_C("otel.env.invalid"), DDOG_METRIC_TYPE_COUNT, DDOG_METRIC_NAMESPACE_TRACERS);
12+
if (!pre_rinit && DDTRACE_G(sidecar) && get_DD_INSTRUMENTATION_TELEMETRY_ENABLED()) {
13+
ddog_sidecar_telemetry_register_metric(&DDTRACE_G(sidecar), DDOG_CHARSLICE_C("otel.env.invalid"), DDOG_METRIC_TYPE_COUNT, DDOG_METRIC_NAMESPACE_TRACERS);
1414
ddog_SidecarActionsBuffer *buffer = ddtrace_telemetry_buffer();
1515
ddog_CharSlice tags;
1616
tags.len = asprintf((char **)&tags.ptr, "config_opentelemetry:%s,config_datadog:%s", otel_cfg, dd_cfg);

0 commit comments

Comments
 (0)