diff --git a/CHANGELOG.md b/CHANGELOG.md index 547af5c275..4f27e7654e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Features**: +- Add HTTP retry with exponential backoff, opt-in via `sentry_options_set_http_retry`. ([#1520](https://github.com/getsentry/sentry-native/pull/1520)) - Store minidump attachments as separate `.dmp` files in the offline cache for direct debugger access. ([#1607](https://github.com/getsentry/sentry-native/pull/1607)) **Fixes**: diff --git a/examples/example.c b/examples/example.c index 4b2eaffcf0..5198a10e2e 100644 --- a/examples/example.c +++ b/examples/example.c @@ -676,6 +676,12 @@ main(int argc, char **argv) sentry_options_set_cache_max_age(options, 5 * 24 * 60 * 60); // 5 days sentry_options_set_cache_max_items(options, 5); } + if (has_arg(argc, argv, "http-retry")) { + sentry_options_set_http_retry(options, true); + } + if (has_arg(argc, argv, "no-http-retry")) { + sentry_options_set_http_retry(options, false); + } if (has_arg(argc, argv, "enable-metrics")) { sentry_options_set_enable_metrics(options, true); diff --git a/include/sentry.h b/include/sentry.h index 073aa77d75..4afeb2be45 100644 --- a/include/sentry.h +++ b/include/sentry.h @@ -946,6 +946,24 @@ SENTRY_API void sentry_transport_set_shutdown_func( sentry_transport_t *transport, int (*shutdown_func)(uint64_t timeout, void *state)); +/** + * Retries sending all pending envelopes in the transport's retry queue, + * e.g. when coming back online. Only applicable for HTTP transports. + * + * Note: The SDK automatically retries failed envelopes on next application + * startup. This function allows manual triggering of pending retries at + * runtime. Each envelope is retried up to 6 times. If all attempts are + * exhausted during intermittent connectivity, events will be discarded + * (or moved to cache if enabled via sentry_options_set_cache_keep). + * + * Warning: This function has no rate limiting - it will immediately + * attempt to send all pending envelopes. Calling this repeatedly during + * extended network outages may exhaust retry attempts that might have + * succeeded with the SDK's built-in exponential backoff. + */ +SENTRY_EXPERIMENTAL_API void sentry_transport_retry( + sentry_transport_t *transport); + /** * Generic way to free transport. */ @@ -2260,6 +2278,18 @@ SENTRY_EXPERIMENTAL_API void sentry_options_set_enable_logs( SENTRY_EXPERIMENTAL_API int sentry_options_get_enable_logs( const sentry_options_t *opts); +/** + * Enables or disables HTTP retry with exponential backoff for network failures. + * + * Only applicable for HTTP transports. + * + * Disabled by default. + */ +SENTRY_EXPERIMENTAL_API void sentry_options_set_http_retry( + sentry_options_t *opts, int enabled); +SENTRY_EXPERIMENTAL_API int sentry_options_get_http_retry( + const sentry_options_t *opts); + /** * Enables or disables custom attributes parsing for structured logging. * diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 218a9b012c..326fd3cebf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,8 @@ sentry_target_sources_cwd(sentry sentry_process.h sentry_ratelimiter.c sentry_ratelimiter.h + sentry_retry.c + sentry_retry.h sentry_ringbuffer.c sentry_ringbuffer.h sentry_sampling_context.h diff --git a/src/backends/native/sentry_crash_daemon.c b/src/backends/native/sentry_crash_daemon.c index ed1b235c25..d0eccc027e 100644 --- a/src/backends/native/sentry_crash_daemon.c +++ b/src/backends/native/sentry_crash_daemon.c @@ -3152,6 +3152,7 @@ sentry__crash_daemon_main(pid_t app_pid, uint64_t app_tid, HANDLE event_handle, sentry_options_set_debug(options, ipc->shmem->debug_enabled); options->attach_screenshot = ipc->shmem->attach_screenshot; options->cache_keep = ipc->shmem->cache_keep; + options->http_retry = false; // Set custom logger that writes to file if (log_file) { diff --git a/src/sentry_core.c b/src/sentry_core.c index e7b86760a8..d53eaeb84f 100644 --- a/src/sentry_core.c +++ b/src/sentry_core.c @@ -292,7 +292,7 @@ sentry_init(sentry_options_t *options) backend->prune_database_func(backend); } - if (options->cache_keep) { + if (options->cache_keep || options->http_retry) { if (!sentry__transport_submit_cleanup(options->transport, options)) { sentry__cleanup_cache(options); } diff --git a/src/sentry_database.c b/src/sentry_database.c index 01cea6e663..fd4f2f14c3 100644 --- a/src/sentry_database.c +++ b/src/sentry_database.c @@ -179,14 +179,120 @@ sentry__run_write_external( bool sentry__run_write_cache( - const sentry_run_t *run, const sentry_envelope_t *envelope) + const sentry_run_t *run, const sentry_envelope_t *envelope, int retry_count) +{ + if (sentry__path_create_dir_all(run->cache_path) != 0) { + SENTRY_ERRORF("mkdir failed: \"%s\"", run->cache_path->path); + return false; + } + + if (retry_count < 0) { + return sentry__envelope_write_to_cache(envelope, run->cache_path) == 0; + } + + sentry_uuid_t event_id = sentry__envelope_get_event_id(envelope); + if (sentry_uuid_is_nil(&event_id)) { + event_id = sentry_uuid_new_v4(); + } + + char uuid[37]; + sentry_uuid_as_string(&event_id, uuid); + + sentry_path_t *path = sentry__run_make_cache_path( + run, sentry__usec_time() / 1000, retry_count, uuid); + if (!path) { + return false; + } + + int rv = sentry_envelope_write_to_path(envelope, path); + sentry__path_free(path); + if (rv) { + SENTRY_WARN("writing envelope to file failed"); + } + return rv == 0; +} + +bool +sentry__parse_cache_filename(const char *filename, uint64_t *ts_out, + int *count_out, const char **uuid_out) +{ + // Minimum retry filename: --.envelope (49+ chars). + // Cache filenames are exactly 45 chars (.envelope). + if (strlen(filename) <= 45) { + return false; + } + + char *end; + uint64_t ts = strtoull(filename, &end, 10); + if (*end != '-') { + return false; + } + + const char *count_str = end + 1; + long count = strtol(count_str, &end, 10); + if (*end != '-' || count < 0) { + return false; + } + + const char *uuid = end + 1; + size_t tail_len = strlen(uuid); + // 36 chars UUID (with dashes) + ".envelope" + if (tail_len != 36 + 9 || strcmp(uuid + 36, ".envelope") != 0) { + return false; + } + + *ts_out = ts; + *count_out = (int)count; + *uuid_out = uuid; + return true; +} + +sentry_path_t * +sentry__run_make_cache_path( + const sentry_run_t *run, uint64_t ts, int count, const char *uuid) +{ + char filename[128]; + if (count >= 0) { + snprintf(filename, sizeof(filename), "%" PRIu64 "-%02d-%.36s.envelope", + ts, count, uuid); + } else { + snprintf(filename, sizeof(filename), "%.36s.envelope", uuid); + } + return sentry__path_join_str(run->cache_path, filename); +} + +bool +sentry__run_move_cache( + const sentry_run_t *run, const sentry_path_t *src, int retry_count) { if (sentry__path_create_dir_all(run->cache_path) != 0) { SENTRY_ERRORF("mkdir failed: \"%s\"", run->cache_path->path); return false; } - return sentry__envelope_write_to_cache(envelope, run->cache_path) == 0; + const char *src_name = sentry__path_filename(src); + uint64_t parsed_ts; + int parsed_count; + const char *parsed_uuid; + const char *cache_name = sentry__parse_cache_filename(src_name, &parsed_ts, + &parsed_count, &parsed_uuid) + ? parsed_uuid + : src_name; + + sentry_path_t *dst_path = sentry__run_make_cache_path( + run, sentry__usec_time() / 1000, retry_count, cache_name); + if (!dst_path) { + return false; + } + + int rv = sentry__path_rename(src, dst_path); + sentry__path_free(dst_path); + if (rv != 0) { + SENTRY_WARNF("failed to cache envelope \"%s\"", src_name); + return false; + } + + return true; } bool diff --git a/src/sentry_database.h b/src/sentry_database.h index 9090857d80..33e6735ceb 100644 --- a/src/sentry_database.h +++ b/src/sentry_database.h @@ -72,10 +72,27 @@ bool sentry__run_clear_session(const sentry_run_t *run); /** * This will serialize and write the given envelope to disk into the cache - * directory: `/cache/.envelope` + * directory. When retry_count >= 0 the filename uses retry format + * `--.envelope`, otherwise `.envelope`. */ -bool sentry__run_write_cache( - const sentry_run_t *run, const sentry_envelope_t *envelope); +bool sentry__run_write_cache(const sentry_run_t *run, + const sentry_envelope_t *envelope, int retry_count); + +/** + * Moves a file into the cache directory. When retry_count >= 0 the + * destination uses retry format `--.envelope`, + * otherwise the original filename is preserved. + */ +bool sentry__run_move_cache( + const sentry_run_t *run, const sentry_path_t *src, int retry_count); + +/** + * Builds a cache path. When count >= 0 the result is + * `/cache/--.envelope`, otherwise + * `/cache/.envelope`. + */ +sentry_path_t *sentry__run_make_cache_path( + const sentry_run_t *run, uint64_t ts, int count, const char *uuid); /** * This function is essential to send crash reports from previous runs of the @@ -92,6 +109,13 @@ bool sentry__run_write_cache( void sentry__process_old_runs( const sentry_options_t *options, uint64_t last_crash); +/** + * Parses a retry cache filename: `--.envelope`. + * Returns false for plain cache filenames (`.envelope`). + */ +bool sentry__parse_cache_filename(const char *filename, uint64_t *ts_out, + int *count_out, const char **uuid_out); + /** * Cleans up the cache based on options.cache_max_items, * options.cache_max_size and options.cache_max_age. diff --git a/src/sentry_options.c b/src/sentry_options.c index d9b4e05d2d..607534ee20 100644 --- a/src/sentry_options.c +++ b/src/sentry_options.c @@ -82,6 +82,7 @@ sentry_options_new(void) opts->crash_reporting_mode = SENTRY_CRASH_REPORTING_MODE_NATIVE_WITH_MINIDUMP; // Default: best of // both worlds + opts->http_retry = false; return opts; } @@ -876,6 +877,18 @@ sentry_options_set_handler_strategy( #endif // SENTRY_PLATFORM_LINUX +void +sentry_options_set_http_retry(sentry_options_t *opts, int enabled) +{ + opts->http_retry = !!enabled; +} + +int +sentry_options_get_http_retry(const sentry_options_t *opts) +{ + return opts->http_retry; +} + void sentry_options_set_propagate_traceparent( sentry_options_t *opts, int propagate_traceparent) diff --git a/src/sentry_options.h b/src/sentry_options.h index 50aeb057b6..064c0d1faf 100644 --- a/src/sentry_options.h +++ b/src/sentry_options.h @@ -78,6 +78,7 @@ struct sentry_options_s { bool enable_metrics; sentry_before_send_metric_function_t before_send_metric_func; void *before_send_metric_data; + bool http_retry; /* everything from here on down are options which are stored here but not exposed through the options API */ diff --git a/src/sentry_retry.c b/src/sentry_retry.c new file mode 100644 index 0000000000..0c22e880e0 --- /dev/null +++ b/src/sentry_retry.c @@ -0,0 +1,344 @@ +#include "sentry_retry.h" +#include "sentry_alloc.h" +#include "sentry_database.h" +#include "sentry_envelope.h" +#include "sentry_logger.h" +#include "sentry_options.h" +#include "sentry_utils.h" + +#include +#include + +#define SENTRY_RETRY_ATTEMPTS 6 +#define SENTRY_RETRY_INTERVAL (15 * 60 * 1000) +#define SENTRY_RETRY_THROTTLE 100 + +typedef enum { + SENTRY_RETRY_STARTUP = 0, + SENTRY_RETRY_RUNNING = 1, + SENTRY_RETRY_SEALED = 2 +} sentry_retry_state_t; + +typedef enum { + SENTRY_POLL_IDLE = 0, + SENTRY_POLL_SCHEDULED = 1, + SENTRY_POLL_SHUTDOWN = 2 +} sentry_poll_state_t; + +struct sentry_retry_s { + sentry_run_t *run; + bool cache_keep; + uint64_t startup_time; + volatile long state; + volatile long scheduled; + sentry_bgworker_t *bgworker; + sentry_retry_send_func_t send_cb; + void *send_data; + sentry_mutex_t sealed_lock; +}; + +sentry_retry_t * +sentry__retry_new(const sentry_options_t *options) +{ + sentry_retry_t *retry = SENTRY_MAKE(sentry_retry_t); + if (!retry) { + return NULL; + } + memset(retry, 0, sizeof(sentry_retry_t)); + sentry__mutex_init(&retry->sealed_lock); + retry->run = sentry__run_incref(options->run); + retry->cache_keep = options->cache_keep; + retry->startup_time = sentry__usec_time() / 1000; + return retry; +} + +void +sentry__retry_free(sentry_retry_t *retry) +{ + if (!retry) { + return; + } + sentry__mutex_free(&retry->sealed_lock); + sentry__run_free(retry->run); + sentry_free(retry); +} + +uint64_t +sentry__retry_backoff(int count) +{ + // TODO: consider adding jitter and shortening the poll interval to spread + // out retries when multiple envelopes (esp. large attachments) pile up. + return (uint64_t)SENTRY_RETRY_INTERVAL << MIN(MAX(count, 0), 5); +} + +typedef struct { + sentry_path_t *path; + uint64_t ts; + int count; + char uuid[37]; +} retry_item_t; + +static int +compare_retry_items(const void *a, const void *b) +{ + const retry_item_t *ia = a; + const retry_item_t *ib = b; + if (ia->ts != ib->ts) { + return ia->ts < ib->ts ? -1 : 1; + } + if (ia->count != ib->count) { + return ia->count - ib->count; + } + return strcmp(ia->uuid, ib->uuid); +} + +static bool +handle_result(sentry_retry_t *retry, const retry_item_t *item, int status_code) +{ + // Only network failures (status_code < 0) trigger retries. HTTP responses + // including 5xx (500, 502, 503, 504) are discarded: + // https://develop.sentry.dev/sdk/foundations/transport/offline-caching/#dealing-with-network-failures + + // network failure with retries remaining: bump count & re-enqueue + if (item->count + 1 < SENTRY_RETRY_ATTEMPTS && status_code < 0) { + sentry_path_t *new_path = sentry__run_make_cache_path(retry->run, + sentry__usec_time() / 1000, item->count + 1, item->uuid); + if (new_path) { + if (sentry__path_rename(item->path, new_path) != 0) { + SENTRY_WARNF( + "failed to rename retry envelope \"%s\"", item->path->path); + } + sentry__path_free(new_path); + } + return true; + } + + bool exhausted = item->count + 1 >= SENTRY_RETRY_ATTEMPTS; + + // network failure with retries exhausted + if (exhausted && status_code < 0) { + if (retry->cache_keep) { + SENTRY_WARNF("max retries (%d) reached, moving envelope to cache", + SENTRY_RETRY_ATTEMPTS); + } else { + SENTRY_WARNF("max retries (%d) reached, discarding envelope", + SENTRY_RETRY_ATTEMPTS); + } + } + + // cache on last attempt + if (exhausted && retry->cache_keep && status_code < 0) { + if (!sentry__run_move_cache(retry->run, item->path, -1)) { + sentry__path_remove(item->path); + } + return false; + } + + sentry__path_remove(item->path); + return false; +} + +size_t +sentry__retry_send(sentry_retry_t *retry, uint64_t before, + sentry_retry_send_func_t send_cb, void *data) +{ + sentry_pathiter_t *piter + = sentry__path_iter_directory(retry->run->cache_path); + if (!piter) { + return 0; + } + + size_t item_cap = 16; + retry_item_t *items = sentry_malloc(item_cap * sizeof(retry_item_t)); + if (!items) { + sentry__pathiter_free(piter); + return 0; + } + + size_t total = 0; + size_t eligible = 0; + uint64_t now = before > 0 ? 0 : sentry__usec_time() / 1000; + + const sentry_path_t *p; + while ((p = sentry__pathiter_next(piter)) != NULL) { + const char *fname = sentry__path_filename(p); + uint64_t ts; + int count; + const char *uuid; + if (!sentry__parse_cache_filename(fname, &ts, &count, &uuid)) { + continue; + } + if (before > 0 && ts >= before) { + continue; + } + total++; + if (!before + && (now < ts || (now - ts) < sentry__retry_backoff(count))) { + continue; + } + if (eligible == item_cap) { + item_cap *= 2; + retry_item_t *tmp = sentry_malloc(item_cap * sizeof(retry_item_t)); + if (!tmp) { + break; + } + memcpy(tmp, items, eligible * sizeof(retry_item_t)); + sentry_free(items); + items = tmp; + } + retry_item_t *item = &items[eligible]; + item->path = sentry__path_clone(p); + if (!item->path) { + break; + } + item->ts = ts; + item->count = count; + memcpy(item->uuid, uuid, 36); + item->uuid[36] = '\0'; + eligible++; + } + sentry__pathiter_free(piter); + + if (eligible > 1) { + qsort(items, eligible, sizeof(retry_item_t), compare_retry_items); + } + + for (size_t i = 0; i < eligible; i++) { + sentry_envelope_t *envelope = sentry__envelope_from_path(items[i].path); + if (!envelope) { + sentry__path_remove(items[i].path); + total--; + } else { + SENTRY_DEBUGF("retrying envelope (%d/%d)", items[i].count + 1, + SENTRY_RETRY_ATTEMPTS); + int status_code = send_cb(envelope, data); + sentry_envelope_free(envelope); + if (!handle_result(retry, &items[i], status_code)) { + total--; + } + // stop on network failure to avoid wasting time on a dead + // connection; remaining envelopes stay untouched for later + if (status_code < 0) { + break; + } + } + } + + for (size_t i = 0; i < eligible; i++) { + sentry__path_free(items[i].path); + } + sentry_free(items); + return total; +} + +static void +retry_poll_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + uint64_t before + = sentry__atomic_fetch(&retry->state) == SENTRY_RETRY_STARTUP + ? retry->startup_time + : 0; + // CAS instead of unconditional store to preserve SENTRY_POLL_SHUTDOWN + sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_SCHEDULED, SENTRY_POLL_IDLE); + if (sentry__retry_send(retry, before, retry->send_cb, retry->send_data) + && sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_IDLE, SENTRY_POLL_SCHEDULED)) { + sentry__bgworker_submit_delayed(retry->bgworker, retry_poll_task, NULL, + retry, SENTRY_RETRY_INTERVAL); + } + // subsequent polls use backoff instead of the startup time filter + sentry__atomic_compare_swap( + &retry->state, SENTRY_RETRY_STARTUP, SENTRY_RETRY_RUNNING); +} + +void +sentry__retry_start(sentry_retry_t *retry, sentry_bgworker_t *bgworker, + sentry_retry_send_func_t send_cb, void *send_data) +{ + retry->bgworker = bgworker; + retry->send_cb = send_cb; + retry->send_data = send_data; + sentry__atomic_store(&retry->scheduled, SENTRY_POLL_SCHEDULED); + sentry__bgworker_submit_delayed( + bgworker, retry_poll_task, NULL, retry, SENTRY_RETRY_THROTTLE); +} + +static void +retry_flush_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + sentry__retry_send( + retry, retry->startup_time, retry->send_cb, retry->send_data); +} + +static bool +drop_task_cb(void *_data, void *_ctx) +{ + (void)_data; + (void)_ctx; + return true; +} + +void +sentry__retry_shutdown(sentry_retry_t *retry) +{ + if (retry) { + // drop the delayed poll and prevent retry_poll_task from re-arming + sentry__bgworker_foreach_matching( + retry->bgworker, retry_poll_task, drop_task_cb, NULL); + sentry__atomic_store(&retry->scheduled, SENTRY_POLL_SHUTDOWN); + sentry__bgworker_submit(retry->bgworker, retry_flush_task, NULL, retry); + } +} + +void +sentry__retry_seal(sentry_retry_t *retry) +{ + if (retry) { + // prevent duplicate writes from a still-running detached worker + sentry__mutex_lock(&retry->sealed_lock); + sentry__atomic_store(&retry->state, SENTRY_RETRY_SEALED); + sentry__mutex_unlock(&retry->sealed_lock); + } +} + +static void +retry_trigger_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + sentry__retry_send(retry, UINT64_MAX, retry->send_cb, retry->send_data); +} + +void +sentry__retry_trigger(sentry_retry_t *retry) +{ + sentry__bgworker_submit(retry->bgworker, retry_trigger_task, NULL, retry); +} + +void +sentry__retry_enqueue(sentry_retry_t *retry, const sentry_envelope_t *envelope) +{ + sentry__mutex_lock(&retry->sealed_lock); + if (sentry__atomic_fetch(&retry->state) == SENTRY_RETRY_SEALED) { + sentry__mutex_unlock(&retry->sealed_lock); + return; + } + if (!sentry__run_write_cache(retry->run, envelope, 0)) { + sentry__mutex_unlock(&retry->sealed_lock); + return; + } + sentry__mutex_unlock(&retry->sealed_lock); + + sentry__atomic_compare_swap( + &retry->state, SENTRY_RETRY_STARTUP, SENTRY_RETRY_RUNNING); + if (sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_IDLE, SENTRY_POLL_SCHEDULED)) { + sentry__bgworker_submit_delayed(retry->bgworker, retry_poll_task, NULL, + retry, SENTRY_RETRY_INTERVAL); + } +} diff --git a/src/sentry_retry.h b/src/sentry_retry.h new file mode 100644 index 0000000000..34ee0e86bf --- /dev/null +++ b/src/sentry_retry.h @@ -0,0 +1,56 @@ +#ifndef SENTRY_RETRY_H_INCLUDED +#define SENTRY_RETRY_H_INCLUDED + +#include "sentry_boot.h" +#include "sentry_path.h" +#include "sentry_sync.h" + +typedef struct sentry_retry_s sentry_retry_t; + +typedef int (*sentry_retry_send_func_t)( + sentry_envelope_t *envelope, void *data); + +sentry_retry_t *sentry__retry_new(const sentry_options_t *options); +void sentry__retry_free(sentry_retry_t *retry); + +/** + * Schedules retry polling on `bgworker` using `send_cb`. + */ +void sentry__retry_start(sentry_retry_t *retry, sentry_bgworker_t *bgworker, + sentry_retry_send_func_t send_cb, void *send_data); + +/** + * Prepares retry for shutdown: drops pending polls and submits a flush task. + */ +void sentry__retry_shutdown(sentry_retry_t *retry); + +/** + * Seals the retry system against further enqueue calls. + */ +void sentry__retry_seal(sentry_retry_t *retry); + +/** + * Writes a failed envelope to the retry dir and schedules a delayed poll. + */ +void sentry__retry_enqueue( + sentry_retry_t *retry, const sentry_envelope_t *envelope); + +/** + * Sends eligible retry files via `send_cb`. `before > 0`: send files with + * ts < before (startup). `before == 0`: use backoff. Returns remaining file + * count for controlling polling. + */ +size_t sentry__retry_send(sentry_retry_t *retry, uint64_t before, + sentry_retry_send_func_t send_cb, void *data); + +/** + * Exponential backoff: 15m, 30m, 1h, 2h, 4h, 8h, 8h, ... (capped at 8h). + */ +uint64_t sentry__retry_backoff(int count); + +/** + * Submits a delayed retry poll task on the background worker. + */ +void sentry__retry_trigger(sentry_retry_t *retry); + +#endif diff --git a/src/sentry_transport.c b/src/sentry_transport.c index 190c4625a5..cc9542293f 100644 --- a/src/sentry_transport.c +++ b/src/sentry_transport.c @@ -10,6 +10,7 @@ struct sentry_transport_s { int (*flush_func)(uint64_t timeout, void *state); void (*free_func)(void *state); size_t (*dump_func)(sentry_run_t *run, void *state); + void (*retry_func)(void *state); void (*cleanup_func)(const sentry_options_t *options, void *state); void *state; bool running; @@ -149,6 +150,21 @@ sentry__transport_get_state(sentry_transport_t *transport) return transport ? transport->state : NULL; } +void +sentry_transport_retry(sentry_transport_t *transport) +{ + if (transport && transport->retry_func) { + transport->retry_func(transport->state); + } +} + +void +sentry__transport_set_retry_func( + sentry_transport_t *transport, void (*retry_func)(void *state)) +{ + transport->retry_func = retry_func; +} + void sentry__transport_set_cleanup_func(sentry_transport_t *transport, void (*cleanup_func)(const sentry_options_t *options, void *state)) diff --git a/src/sentry_transport.h b/src/sentry_transport.h index f03274979f..a4ce8197ed 100644 --- a/src/sentry_transport.h +++ b/src/sentry_transport.h @@ -57,6 +57,9 @@ size_t sentry__transport_dump_queue( void *sentry__transport_get_state(sentry_transport_t *transport); +void sentry__transport_set_retry_func( + sentry_transport_t *transport, void (*retry_func)(void *state)); + /** * Sets the cleanup function of the transport. * diff --git a/src/transports/sentry_http_transport.c b/src/transports/sentry_http_transport.c index 6449e52908..a40843e7bb 100644 --- a/src/transports/sentry_http_transport.c +++ b/src/transports/sentry_http_transport.c @@ -4,6 +4,7 @@ #include "sentry_envelope.h" #include "sentry_options.h" #include "sentry_ratelimiter.h" +#include "sentry_retry.h" #include "sentry_string.h" #include "sentry_transport.h" #include "sentry_utils.h" @@ -30,6 +31,7 @@ typedef struct { int (*start_client)(void *, const sentry_options_t *); sentry_http_send_func_t send_func; void (*shutdown_client)(void *client); + sentry_retry_t *retry; bool cache_keep; sentry_run_t *run; } http_transport_state_t; @@ -214,8 +216,9 @@ http_send_request( } static int -http_send_envelope(http_transport_state_t *state, sentry_envelope_t *envelope) +http_send_envelope(sentry_envelope_t *envelope, void *_state) { + http_transport_state_t *state = _state; sentry_prepared_http_request_t *req = sentry__prepare_http_request( envelope, state->dsn, state->ratelimiter, state->user_agent); if (!req) { @@ -236,6 +239,7 @@ http_transport_state_free(void *_state) sentry__dsn_decref(state->dsn); sentry_free(state->user_agent); sentry__rate_limiter_free(state->ratelimiter); + sentry__retry_free(state->retry); sentry__run_free(state->run); sentry_free(state); } @@ -246,9 +250,11 @@ http_send_task(void *_envelope, void *_state) sentry_envelope_t *envelope = _envelope; http_transport_state_t *state = _state; - int status_code = http_send_envelope(state, envelope); - if (status_code < 0 && state->cache_keep) { - sentry__run_write_cache(state->run, envelope); + int status_code = http_send_envelope(envelope, state); + if (status_code < 0 && state->retry) { + sentry__retry_enqueue(state->retry, envelope); + } else if (status_code < 0 && state->cache_keep) { + sentry__run_write_cache(state->run, envelope, -1); } } @@ -289,7 +295,20 @@ http_transport_start(const sentry_options_t *options, void *transport_state) } } - return sentry__bgworker_start(bgworker); + int rv = sentry__bgworker_start(bgworker); + if (rv != 0) { + return rv; + } + + if (options->http_retry) { + state->retry = sentry__retry_new(options); + if (state->retry) { + sentry__retry_start( + state->retry, bgworker, http_send_envelope, state); + } + } + + return 0; } static int @@ -312,11 +331,14 @@ http_transport_shutdown(uint64_t timeout, void *transport_state) sentry_bgworker_t *bgworker = transport_state; http_transport_state_t *state = sentry__bgworker_get_state(bgworker); + sentry__retry_shutdown(state->retry); + int rv = sentry__bgworker_shutdown_cb( bgworker, timeout, http_transport_shutdown_timeout, state); if (rv != 0) { sentry__bgworker_foreach_matching( bgworker, http_cleanup_cache_task, http_flush_cleanup_cb, NULL); + sentry__retry_seal(state->retry); } return rv; } @@ -352,6 +374,16 @@ http_transport_get_state(sentry_transport_t *transport) return sentry__bgworker_get_state(bgworker); } +static void +http_transport_retry(void *transport_state) +{ + sentry_bgworker_t *bgworker = transport_state; + http_transport_state_t *state = sentry__bgworker_get_state(bgworker); + if (state->retry) { + sentry__retry_trigger(state->retry); + } +} + static void http_transport_submit_cleanup( const sentry_options_t *options, void *transport_state) @@ -395,6 +427,7 @@ sentry__http_transport_new(void *client, sentry_http_send_func_t send_func) sentry_transport_set_flush_func(transport, http_transport_flush); sentry_transport_set_shutdown_func(transport, http_transport_shutdown); sentry__transport_set_dump_func(transport, http_dump_queue); + sentry__transport_set_retry_func(transport, http_transport_retry); sentry__transport_set_cleanup_func( transport, http_transport_submit_cleanup); diff --git a/tests/test_integration_cache.py b/tests/test_integration_cache.py index 9b5310ca45..50bc6adaf3 100644 --- a/tests/test_integration_cache.py +++ b/tests/test_integration_cache.py @@ -35,7 +35,8 @@ def test_cache_keep(cmake, backend, cache_keep): run( tmp_path, "sentry_example", - ["log", "flush", "crash"] + (["cache-keep"] if cache_keep else []), + ["log", "no-http-retry", "flush", "crash"] + + (["cache-keep"] if cache_keep else []), expect_failure=True, env=env, ) @@ -46,7 +47,8 @@ def test_cache_keep(cmake, backend, cache_keep): run( tmp_path, "sentry_example", - ["log", "flush", "no-setup"] + (["cache-keep"] if cache_keep else []), + ["log", "no-http-retry", "flush", "no-setup"] + + (["cache-keep"] if cache_keep else []), env=env, ) @@ -81,7 +83,7 @@ def test_cache_max_size(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "crash"], + ["log", "no-http-retry", "cache-keep", "flush", "crash"], expect_failure=True, env=env, ) @@ -90,7 +92,7 @@ def test_cache_max_size(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "no-setup"], + ["log", "no-http-retry", "cache-keep", "flush", "no-setup"], env=env, ) @@ -105,7 +107,7 @@ def test_cache_max_size(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "no-setup"], + ["log", "no-http-retry", "cache-keep", "no-setup"], env=env, ) @@ -135,7 +137,7 @@ def test_cache_max_age(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "crash"], + ["log", "no-http-retry", "cache-keep", "flush", "crash"], expect_failure=True, env=env, ) @@ -144,7 +146,7 @@ def test_cache_max_age(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "no-setup"], + ["log", "no-http-retry", "cache-keep", "flush", "no-setup"], env=env, ) @@ -159,7 +161,7 @@ def test_cache_max_age(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "no-setup"], + ["log", "no-http-retry", "cache-keep", "no-setup"], env=env, ) @@ -190,7 +192,7 @@ def test_cache_max_items(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "crash"], + ["log", "no-http-retry", "cache-keep", "flush", "crash"], expect_failure=True, env=env, ) @@ -199,7 +201,7 @@ def test_cache_max_items(cmake, backend): run( tmp_path, "sentry_example", - ["log", "cache-keep", "flush", "no-setup"], + ["log", "no-http-retry", "cache-keep", "flush", "no-setup"], env=env, ) @@ -207,3 +209,59 @@ def test_cache_max_items(cmake, backend): assert cache_dir.exists() cache_files = list(cache_dir.glob("*.envelope")) assert len(cache_files) == 5 + + +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad, reason="breakpad backend not available" + ), + ), + ], +) +def test_cache_max_items_with_retry(cmake, backend): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": backend}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + # Create cache files via crash+restart cycles + for i in range(4): + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + env=env, + ) + + # flush + cache + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "flush", "no-setup"], + env=env, + ) + + # Pre-populate cache/ with retry-format envelope files + cache_dir.mkdir(parents=True, exist_ok=True) + for i in range(4): + ts = int(time.time() * 1000) + f = cache_dir / f"{ts}-00-00000000-0000-0000-0000-{i:012x}.envelope" + f.write_text("dummy envelope content") + + # Trigger sentry_init which runs cleanup + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + # max 5 items total in cache/ + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) <= 5 diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index 5d6786a2f3..5aad5c1d88 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -59,6 +59,8 @@ def get_asan_crash_env(env): pytestmark = pytest.mark.skipif(not has_http, reason="tests need http") +unreachable_dsn = "http://uiaeosnrtdy@127.0.0.1:19999/123456" + # fmt: off auth_header = ( f"Sentry sentry_key=uiaeosnrtdy, sentry_version=7, sentry_client=sentry.native/{SENTRY_VERSION}" @@ -839,3 +841,363 @@ def test_native_crash_http(cmake, httpserver): assert_minidump(envelope) assert_breadcrumb(envelope) assert_attachment(envelope) + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_on_network_error(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + # unreachable port triggers CURLE_COULDNT_CONNECT + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "capture-event"], + env=dict(os.environ, SENTRY_DSN=unreachable_dsn), + ) + + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + + # retry on next run with working server + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + assert waiting.result + + assert len(httpserver.log) == 1 + envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + assert envelope.headers["event_id"] == envelope_uuid + assert_meta(envelope, integration="inproc") + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_attempts(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run(tmp_path, "sentry_example", ["log", "http-retry", "capture-event"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + envelope = Envelope.deserialize(cache_files[0].read_bytes()) + assert envelope.headers["event_id"] == envelope_uuid + + run(tmp_path, "sentry_example", ["log", "http-retry", "no-setup"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-01-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + run(tmp_path, "sentry_example", ["log", "http-retry", "no-setup"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-02-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + # exhaust remaining retries (max 6) + for i in range(4): + run(tmp_path, "sentry_example", ["log", "http-retry", "no-setup"], env=env) + + # discarded after max retries (cache_keep not enabled) + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_with_cache_keep(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "cache-keep", "capture-event"], + env=dict(os.environ, SENTRY_DSN=unreachable_dsn), + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "cache-keep", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + assert waiting.result + + assert len(list(cache_dir.glob("*.envelope"))) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_cache_keep_max_attempts(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "cache-keep", "capture-event"], + env=env, + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + for _ in range(5): + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "cache-keep", "no-setup"], + env=env, + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + # last attempt succeeds — envelope should be removed, not cached + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "cache-keep", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + assert waiting.result + + assert len(list(cache_dir.glob("*.envelope"))) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_http_error_discards_envelope(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "Internal Server Error", status=500 + ) + + with httpserver.wait(timeout=10) as waiting: + run(tmp_path, "sentry_example", ["log", "http-retry", "capture-event"], env=env) + assert waiting.result + + # HTTP errors discard, not retry + cache_files = list(cache_dir.glob("*.envelope")) if cache_dir.exists() else [] + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_rate_limit_discards_envelope(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "Rate Limited", status=429, headers={"retry-after": "60"} + ) + + with httpserver.wait(timeout=10) as waiting: + run(tmp_path, "sentry_example", ["log", "http-retry", "capture-event"], env=env) + assert waiting.result + + # 429 discards, not retry + cache_files = list(cache_dir.glob("*.envelope")) if cache_dir.exists() else [] + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_success(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + db_dir = tmp_path.joinpath(".sentry-native") + cache_dir = db_dir.joinpath("cache") + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "capture-multiple"], + env=dict(os.environ, SENTRY_DSN=unreachable_dsn), + ) + + # envelopes end up in cache/ (retry) or *.run/ (dumped on shutdown timeout) + cached = list(cache_dir.glob("*.envelope")) + dumped = list(db_dir.glob("*.run/*.envelope")) + assert len(cached) + len(dumped) == 10 + + for _ in range(10): + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "OK" + ) + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + assert waiting.result + + assert len(httpserver.log) == 10 + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_network_error(cmake): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + db_dir = tmp_path.joinpath(".sentry-native") + cache_dir = db_dir.joinpath("cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "capture-multiple"], + env=env, + ) + + # envelopes end up in cache/ (retry) or *.run/ (dumped on shutdown timeout) + cached = list(cache_dir.glob("*.envelope")) + dumped = list(db_dir.glob("*.run/*.envelope")) + assert len(cached) + len(dumped) == 10 + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=env, + ) + + # envelopes end up in cache/ (retry) or *.run/ (dumped on shutdown timeout) + cached = list(cache_dir.glob("*.envelope")) + dumped = list(db_dir.glob("*.run/*.envelope")) + assert len(cached) + len(dumped) == 10 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_rate_limit(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + db_dir = tmp_path.joinpath(".sentry-native") + cache_dir = db_dir.joinpath("cache") + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "capture-multiple"], + env=dict(os.environ, SENTRY_DSN=unreachable_dsn), + ) + + # envelopes end up in cache/ (retry) or *.run/ (dumped on shutdown timeout) + cached = list(cache_dir.glob("*.envelope")) + dumped = list(db_dir.glob("*.run/*.envelope")) + assert len(cached) + len(dumped) == 10 + + # rate limit response followed by discards for the rest (rate limiter + # kicks in after the first 429) + httpserver.expect_request("/api/123456/envelope/").respond_with_data( + "Rate Limited", status=429, headers={"retry-after": "60"} + ) + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + + # first envelope gets 429, rest are discarded by rate limiter + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_session_on_network_error(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "start-session"], + env=env, + ) + + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + + # second and third attempts still fail — envelope gets renamed each time + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=env, + ) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-01-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=env, + ) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-02-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + # succeed on fourth attempt + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "http-retry", "no-setup"], + env=dict(os.environ, SENTRY_DSN=make_dsn(httpserver)), + ) + assert waiting.result + + assert len(httpserver.log) == 1 + envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + assert_session(envelope, {"init": True, "status": "exited", "errors": 0}) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index fd16affcac..3b3036259b 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -43,6 +43,7 @@ add_executable(sentry_test_unit test_path.c test_process.c test_ratelimiter.c + test_retry.c test_ringbuffer.c test_sampling.c test_scope.c diff --git a/tests/unit/test_cache.c b/tests/unit/test_cache.c index 57d5a171ba..f8e0083f70 100644 --- a/tests/unit/test_cache.c +++ b/tests/unit/test_cache.c @@ -48,6 +48,7 @@ SENTRY_TEST(cache_keep) TEST_ASSERT(!!options->transport); sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); sentry_options_set_cache_keep(options, true); + sentry_options_set_http_retry(options, false); sentry_init(options); sentry_path_t *cache_path @@ -246,6 +247,69 @@ SENTRY_TEST(cache_max_items) sentry_close(); } +SENTRY_TEST(cache_max_items_with_retry) +{ +#if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) + SKIP_TEST(); +#endif + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_items(options, 7); + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(cache_path) == 0); + + time_t now = time(NULL); + + // 5 cache-format files: 1,3,5,7,9 min old + for (int i = 0; i < 5; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char *filename = sentry__uuid_as_filename(&event_id, ".envelope"); + TEST_ASSERT(!!filename); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + sentry_free(filename); + + TEST_ASSERT(sentry__path_touch(filepath) == 0); + TEST_ASSERT(set_file_mtime(filepath, now - ((i * 2 + 1) * 60)) == 0); + sentry__path_free(filepath); + } + + // 5 retry-format files: 0,2,4,6,8 min old + for (int i = 0; i < 5; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char uuid[37]; + sentry_uuid_as_string(&event_id, uuid); + char filename[128]; + snprintf(filename, sizeof(filename), "%" PRIu64 "-00-%.36s.envelope", + (uint64_t)now, uuid); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + + TEST_ASSERT(sentry__path_touch(filepath) == 0); + TEST_ASSERT(set_file_mtime(filepath, now - (i * 2 * 60)) == 0); + sentry__path_free(filepath); + } + + sentry__cleanup_cache(options); + + int total_count = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(cache_path); + const sentry_path_t *entry; + while (iter && (entry = sentry__pathiter_next(iter)) != NULL) { + total_count++; + } + sentry__pathiter_free(iter); + + TEST_CHECK_INT_EQUAL(total_count, 7); + + sentry__path_free(cache_path); + sentry_close(); +} + SENTRY_TEST(cache_max_size_and_age) { #if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) @@ -345,7 +409,7 @@ SENTRY_TEST(cache_write_minidump) sentry__envelope_item_set_header( item, "filename", sentry_value_new_string("minidump.dmp")); - TEST_CHECK(sentry__run_write_cache(options->run, envelope)); + TEST_CHECK(sentry__run_write_cache(options->run, envelope, -1)); sentry_envelope_free(envelope); char *env_filename = sentry__uuid_as_filename(&event_id, ".envelope"); @@ -436,7 +500,7 @@ SENTRY_TEST(cache_write_raw_with_minidump) TEST_ASSERT(!!raw); // write raw envelope to cache — should materialize and split - TEST_CHECK(sentry__run_write_cache(options->run, raw)); + TEST_CHECK(sentry__run_write_cache(options->run, raw, -1)); sentry_envelope_free(raw); char *dmp_filename = sentry__uuid_as_filename(&event_id, ".dmp"); diff --git a/tests/unit/test_retry.c b/tests/unit/test_retry.c new file mode 100644 index 0000000000..77cc1a993c --- /dev/null +++ b/tests/unit/test_retry.c @@ -0,0 +1,491 @@ +#include "sentry_core.h" +#include "sentry_database.h" +#include "sentry_envelope.h" +#include "sentry_options.h" +#include "sentry_path.h" +#include "sentry_retry.h" +#include "sentry_scope.h" +#include "sentry_session.h" +#include "sentry_testsupport.h" +#include "sentry_transport.h" +#include "sentry_utils.h" +#include "sentry_uuid.h" + +#include + +static int +count_envelope_files(const sentry_path_t *dir) +{ + int count = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(dir); + const sentry_path_t *file; + while (iter && (file = sentry__pathiter_next(iter)) != NULL) { + if (sentry__path_ends_with(file, ".envelope")) { + count++; + } + } + sentry__pathiter_free(iter); + return count; +} + +static int +find_envelope_attempt(const sentry_path_t *dir) +{ + sentry_pathiter_t *iter = sentry__path_iter_directory(dir); + const sentry_path_t *file; + while (iter && (file = sentry__pathiter_next(iter)) != NULL) { + if (!sentry__path_ends_with(file, ".envelope")) { + continue; + } + const char *name = sentry__path_filename(file); + uint64_t ts; + int attempt; + const char *uuid; + if (sentry__parse_cache_filename(name, &ts, &attempt, &uuid)) { + sentry__pathiter_free(iter); + return attempt; + } + } + sentry__pathiter_free(iter); + return -1; +} + +static void +write_retry_file(const sentry_run_t *run, uint64_t timestamp, int retry_count, + const sentry_uuid_t *event_id) +{ + sentry_envelope_t *envelope = sentry__envelope_new(); + sentry_value_t event = sentry__value_new_event_with_id(event_id); + sentry__envelope_add_event(envelope, event); + + char uuid[37]; + sentry_uuid_as_string(event_id, uuid); + + sentry_path_t *path + = sentry__run_make_cache_path(run, timestamp, retry_count, uuid); + (void)sentry_envelope_write_to_path(envelope, path); + sentry__path_free(path); + sentry_envelope_free(envelope); +} + +typedef struct { + int status_code; + size_t count; +} retry_test_ctx_t; + +static int +test_send_cb(sentry_envelope_t *envelope, void *_ctx) +{ + (void)envelope; + retry_test_ctx_t *ctx = _ctx; + ctx->count++; + return ctx->status_code; +} + +SENTRY_TEST(retry_filename) +{ + uint64_t ts; + int count; + const char *uuid; + + TEST_CHECK(sentry__parse_cache_filename( + "1234567890-00-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, + &count, &uuid)); + TEST_CHECK_UINT64_EQUAL(ts, 1234567890); + TEST_CHECK_INT_EQUAL(count, 0); + TEST_CHECK(strncmp(uuid, "abcdefab-1234-5678-9abc-def012345678", 36) == 0); + + TEST_CHECK(sentry__parse_cache_filename( + "999-04-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, + &uuid)); + TEST_CHECK_UINT64_EQUAL(ts, 999); + TEST_CHECK_INT_EQUAL(count, 4); + + // negative count + TEST_CHECK(!sentry__parse_cache_filename( + "123--01-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, + &uuid)); + + // cache filename (no timestamp/count) + TEST_CHECK(!sentry__parse_cache_filename( + "abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, &uuid)); + + // missing .envelope suffix + TEST_CHECK(!sentry__parse_cache_filename( + "123-00-abcdefab-1234-5678-9abc-def012345678.txt", &ts, &count, &uuid)); +} + +SENTRY_TEST(retry_make_cache_path) +{ +#if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) + SKIP_TEST(); +#endif + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + const char *uuid = "abcdefab-1234-5678-9abc-def012345678"; + + // count >= 0 → retry format + sentry_path_t *path + = sentry__run_make_cache_path(options->run, 1000, 2, uuid); + TEST_CHECK_STRING_EQUAL(sentry__path_filename(path), + "1000-02-abcdefab-1234-5678-9abc-def012345678.envelope"); + sentry__path_free(path); + + // count < 0 → cache format + path = sentry__run_make_cache_path(options->run, 0, -1, uuid); + TEST_CHECK_STRING_EQUAL(sentry__path_filename(path), + "abcdefab-1234-5678-9abc-def012345678.envelope"); + sentry__path_free(path); + + sentry_close(); +} + +SENTRY_TEST(retry_throttle) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t ids[4]; + for (int i = 0; i < 4; i++) { + ids[i] = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 0, &ids[i]); + } + + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 4); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 4); + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_skew) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + // future timestamp simulates clock moving backward + uint64_t future_ts = sentry__usec_time() / 1000 + 1000000; + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, future_ts, 0, &event_id); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + + // item should NOT be processed due to backoff (clock backward) + TEST_CHECK_INT_EQUAL(ctx.count, 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_result) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + + // 1. Success (200) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 0); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 2. Rate limited (429) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + ctx = (retry_test_ctx_t) { 429, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 3. Discard (0) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + ctx = (retry_test_ctx_t) { 0, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 4. Network error → bumps count + write_retry_file(options->run, old_ts, 0, &event_id); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 0); + + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 1); + + // 5. Network error at last attempt → removed + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + uint64_t very_old_ts + = sentry__usec_time() / 1000 - 2 * sentry__retry_backoff(5); + write_retry_file(options->run, very_old_ts, 5, &event_id); + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_session) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_release(options, "test@1.0.0"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + sentry_session_t *session = NULL; + SENTRY_WITH_SCOPE (scope) { + session = sentry__session_new(scope); + } + TEST_ASSERT(!!session); + sentry_envelope_t *envelope = sentry__envelope_new(); + TEST_ASSERT(!!envelope); + sentry__envelope_add_session(envelope, session); + + TEST_CHECK(sentry__run_write_cache(options->run, envelope, 0)); + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 1); + + sentry_envelope_free(envelope); + sentry__session_free(session); + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_cache) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_options_set_cache_keep(options, 1); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts = sentry__usec_time() / 1000 - 2 * sentry__retry_backoff(5); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 5, &event_id); + + char uuid_str[37]; + sentry_uuid_as_string(&event_id, uuid_str); + char cache_name[46]; + snprintf(cache_name, sizeof(cache_name), "%.36s.envelope", uuid_str); + sentry_path_t *cached = sentry__path_join_str(cache_path, cache_name); + + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK(!sentry__path_is_file(cached)); + + // Network error on a file at count=5 with max_retries=6 → renames to + // cache format (.envelope) + retry_test_ctx_t ctx = { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK(sentry__path_is_file(cached)); + + // Success on a file at count=5 → removed (successfully delivered) + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + write_retry_file(options->run, old_ts, 5, &event_id); + TEST_CHECK(!sentry__path_is_file(cached)); + + ctx = (retry_test_ctx_t) { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + sentry__retry_free(retry); + sentry__path_free(cached); + sentry_close(); +} + +static int retry_func_calls = 0; + +static void +mock_retry_func(void *state) +{ + (void)state; + retry_func_calls++; +} + +static void +noop_send(sentry_envelope_t *envelope, void *state) +{ + (void)state; + sentry_envelope_free(envelope); +} + +SENTRY_TEST(transport_retry) +{ + // no retry_func → no-op + sentry_transport_t *transport = sentry_transport_new(noop_send); + retry_func_calls = 0; + sentry_transport_retry(transport); + TEST_CHECK_INT_EQUAL(retry_func_calls, 0); + + // with retry_func → calls it + sentry__transport_set_retry_func(transport, mock_retry_func); + sentry_transport_retry(transport); + TEST_CHECK_INT_EQUAL(retry_func_calls, 1); + + // NULL transport → no-op + sentry_transport_retry(NULL); + TEST_CHECK_INT_EQUAL(retry_func_calls, 1); + + sentry_transport_free(transport); +} + +SENTRY_TEST(retry_backoff) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t base = sentry__retry_backoff(0); + uint64_t ref = sentry__usec_time() / 1000 - 10 * base; + + // retry 0: 10*base old, eligible (backoff=base) + sentry_uuid_t id1 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref, 0, &id1); + + // retry 1: 1*base old, not yet eligible (backoff=2*base) + sentry_uuid_t id2 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref + 9 * base, 1, &id2); + + // retry 1: 10*base old, eligible (backoff=2*base) + sentry_uuid_t id3 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref, 1, &id3); + + // retry 2: 2*base old, not eligible (backoff=4*base) + sentry_uuid_t id4 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref + 8 * base, 2, &id4); + + // With backoff: only eligible ones (id1 and id3) are sent + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 2); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 2); + + // Startup scan (no backoff check): remaining 2 files are sent + ctx = (retry_test_ctx_t) { 200, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 2); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // Verify backoff calculation + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(0), base); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(1), base * 2); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(2), base * 4); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(3), base * 8); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(4), base * 16); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(5), base * 32); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(6), base * 32); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(-1), base); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_trigger) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 0, &event_id); + + // UINT64_MAX (trigger mode) bypasses backoff: bumps count + retry_test_ctx_t ctx = { -1, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 1); + + // second call: bumps again because UINT64_MAX skips backoff + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 2); + + // before=0 (poll mode) respects backoff: item is skipped + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 0); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 2); + + sentry__retry_free(retry); + sentry_close(); +} diff --git a/tests/unit/tests.inc b/tests/unit/tests.inc index df781a7195..16791d9567 100644 --- a/tests/unit/tests.inc +++ b/tests/unit/tests.inc @@ -43,6 +43,7 @@ XX(build_id_parser) XX(cache_keep) XX(cache_max_age) XX(cache_max_items) +XX(cache_max_items_with_retry) XX(cache_max_size) XX(cache_max_size_and_age) XX(cache_write_minidump) @@ -194,6 +195,15 @@ XX(read_envelope_from_file) XX(read_write_envelope_to_file_null) XX(read_write_envelope_to_invalid_path) XX(recursive_paths) +XX(retry_backoff) +XX(retry_cache) +XX(retry_filename) +XX(retry_make_cache_path) +XX(retry_result) +XX(retry_session) +XX(retry_skew) +XX(retry_throttle) +XX(retry_trigger) XX(ringbuffer_append) XX(ringbuffer_append_invalid_decref_value) XX(ringbuffer_append_null_decref_value) @@ -243,6 +253,7 @@ XX(traceparent_header_disabled_by_default) XX(traceparent_header_generation) XX(transaction_name_backfill_on_finish) XX(transactions_skip_before_send) +XX(transport_retry) XX(transport_sampling_transactions) XX(transport_sampling_transactions_set_trace) XX(txn_data)