diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dc886807..6fcc90b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Features**: +- Add HTTP retry with exponential backoff. ([#1520](https://github.com/getsentry/sentry-native/pull/1520)) - Support `SENTRY_SAMPLE_RATE` and `SENTRY_TRACES_SAMPLE_RATE` environment variables. ([#1540](https://github.com/getsentry/sentry-native/pull/1540)) **Fixes**: diff --git a/examples/example.c b/examples/example.c index 70ce402c7..72d80cef0 100644 --- a/examples/example.c +++ b/examples/example.c @@ -633,6 +633,9 @@ main(int argc, char **argv) sentry_options_set_cache_max_age(options, 5 * 24 * 60 * 60); // 5 days sentry_options_set_cache_max_items(options, 5); } + if (has_arg(argc, argv, "no-http-retry")) { + sentry_options_set_http_retry(options, false); + } if (has_arg(argc, argv, "enable-metrics")) { sentry_options_set_enable_metrics(options, true); diff --git a/include/sentry.h b/include/sentry.h index 50d30e3f2..618a26d5b 100644 --- a/include/sentry.h +++ b/include/sentry.h @@ -946,6 +946,24 @@ SENTRY_API void sentry_transport_set_shutdown_func( sentry_transport_t *transport, int (*shutdown_func)(uint64_t timeout, void *state)); +/** + * Retries sending all pending envelopes in the transport's retry queue, + * e.g. when coming back online. Only applicable for HTTP transports. + * + * Note: The SDK automatically retries failed envelopes on next application + * startup. This function allows manual triggering of pending retries at + * runtime. Each envelope is retried up to 6 times. If all attempts are + * exhausted during intermittent connectivity, events will be discarded + * (or moved to cache if enabled via sentry_options_set_cache_keep). + * + * Warning: This function has no rate limiting - it will immediately + * attempt to send all pending envelopes. Calling this repeatedly during + * extended network outages may exhaust retry attempts that might have + * succeeded with the SDK's built-in exponential backoff. + */ +SENTRY_EXPERIMENTAL_API void sentry_transport_retry( + sentry_transport_t *transport); + /** * Generic way to free transport. */ @@ -2146,6 +2164,15 @@ SENTRY_EXPERIMENTAL_API void sentry_options_set_enable_logs( SENTRY_EXPERIMENTAL_API int sentry_options_get_enable_logs( const sentry_options_t *opts); +/** + * Enables or disables HTTP retry with exponential backoff for network failures. + * Enabled by default. + */ +SENTRY_EXPERIMENTAL_API void sentry_options_set_http_retry( + sentry_options_t *opts, int enabled); +SENTRY_EXPERIMENTAL_API int sentry_options_get_http_retry( + const sentry_options_t *opts); + /** * Enables or disables custom attributes parsing for structured logging. * diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d488ae4cd..87098e2f3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,8 @@ sentry_target_sources_cwd(sentry sentry_process.h sentry_ratelimiter.c sentry_ratelimiter.h + sentry_retry.c + sentry_retry.h sentry_ringbuffer.c sentry_ringbuffer.h sentry_sampling_context.h diff --git a/src/backends/sentry_backend_crashpad.cpp b/src/backends/sentry_backend_crashpad.cpp index 9815bdb42..ba7420734 100644 --- a/src/backends/sentry_backend_crashpad.cpp +++ b/src/backends/sentry_backend_crashpad.cpp @@ -565,11 +565,9 @@ process_completed_reports( SENTRY_DEBUGF("caching %zu completed reports", reports.size()); - sentry_path_t *cache_dir - = sentry__path_join_str(options->database_path, "cache"); - if (!cache_dir || sentry__path_create_dir_all(cache_dir) != 0) { + sentry_path_t *cache_dir = options->run->cache_path; + if (sentry__path_create_dir_all(cache_dir) != 0) { SENTRY_WARN("failed to create cache dir"); - sentry__path_free(cache_dir); return; } @@ -593,8 +591,6 @@ process_completed_reports( sentry__path_free(out_path); sentry_envelope_free(envelope); } - - sentry__path_free(cache_dir); } static int diff --git a/src/sentry_core.c b/src/sentry_core.c index 1c70c9ecc..135acd1e9 100644 --- a/src/sentry_core.c +++ b/src/sentry_core.c @@ -290,7 +290,7 @@ sentry_init(sentry_options_t *options) backend->prune_database_func(backend); } - if (options->cache_keep) { + if (options->cache_keep || options->http_retry) { sentry__cleanup_cache(options); } diff --git a/src/sentry_database.c b/src/sentry_database.c index 45f8b8eb1..a7114e3bc 100644 --- a/src/sentry_database.c +++ b/src/sentry_database.c @@ -4,6 +4,9 @@ #include "sentry_json.h" #include "sentry_options.h" #include "sentry_session.h" +#include "sentry_sync.h" +#include "sentry_transport.h" +#include "sentry_utils.h" #include "sentry_uuid.h" #include #include @@ -50,19 +53,32 @@ sentry__run_new(const sentry_path_t *database_path) return NULL; } + // `/cache` + sentry_path_t *cache_path = sentry__path_join_str(database_path, "cache"); + if (!cache_path) { + sentry__path_free(run_path); + sentry__path_free(lock_path); + sentry__path_free(session_path); + sentry__path_free(external_path); + return NULL; + } + sentry_run_t *run = SENTRY_MAKE(sentry_run_t); if (!run) { sentry__path_free(run_path); sentry__path_free(session_path); sentry__path_free(lock_path); sentry__path_free(external_path); + sentry__path_free(cache_path); return NULL; } + run->refcount = 1; run->uuid = uuid; run->run_path = run_path; run->session_path = session_path; run->external_path = external_path; + run->cache_path = cache_path; run->lock = sentry__filelock_new(lock_path); if (!run->lock) { goto error; @@ -80,6 +96,15 @@ sentry__run_new(const sentry_path_t *database_path) return NULL; } +sentry_run_t * +sentry__run_incref(sentry_run_t *run) +{ + if (run) { + sentry__atomic_fetch_and_add(&run->refcount, 1); + } + return run; +} + void sentry__run_clean(sentry_run_t *run) { @@ -90,18 +115,20 @@ sentry__run_clean(sentry_run_t *run) void sentry__run_free(sentry_run_t *run) { - if (!run) { + if (!run || sentry__atomic_fetch_and_add(&run->refcount, -1) != 1) { return; } sentry__path_free(run->run_path); sentry__path_free(run->session_path); sentry__path_free(run->external_path); + sentry__path_free(run->cache_path); sentry__filelock_free(run->lock); sentry_free(run); } static bool -write_envelope(const sentry_path_t *path, const sentry_envelope_t *envelope) +write_envelope(const sentry_path_t *path, const sentry_envelope_t *envelope, + int retry_count) { sentry_uuid_t event_id = sentry__envelope_get_event_id(envelope); @@ -111,13 +138,18 @@ write_envelope(const sentry_path_t *path, const sentry_envelope_t *envelope) event_id = sentry_uuid_new_v4(); } - char *envelope_filename = sentry__uuid_as_filename(&event_id, ".envelope"); - if (!envelope_filename) { - return false; + char uuid[37]; + sentry_uuid_as_string(&event_id, uuid); + + char filename[128]; + if (retry_count >= 0) { + snprintf(filename, sizeof(filename), "%" PRIu64 "-%02d-%.36s.envelope", + sentry__usec_time() / 1000, retry_count, uuid); + } else { + snprintf(filename, sizeof(filename), "%.36s.envelope", uuid); } - sentry_path_t *output_path = sentry__path_join_str(path, envelope_filename); - sentry_free(envelope_filename); + sentry_path_t *output_path = sentry__path_join_str(path, filename); if (!output_path) { return false; } @@ -136,7 +168,7 @@ bool sentry__run_write_envelope( const sentry_run_t *run, const sentry_envelope_t *envelope) { - return write_envelope(run->run_path, envelope); + return write_envelope(run->run_path, envelope, -1); } bool @@ -148,7 +180,104 @@ sentry__run_write_external( return false; } - return write_envelope(run->external_path, envelope); + return write_envelope(run->external_path, envelope, -1); +} + +bool +sentry__run_write_cache( + const sentry_run_t *run, const sentry_envelope_t *envelope, int retry_count) +{ + if (sentry__path_create_dir_all(run->cache_path) != 0) { + SENTRY_ERRORF("mkdir failed: \"%s\"", run->cache_path->path); + return false; + } + + return write_envelope(run->cache_path, envelope, retry_count); +} + +bool +sentry__parse_cache_filename(const char *filename, uint64_t *ts_out, + int *count_out, const char **uuid_out) +{ + // Minimum retry filename: --.envelope (49+ chars). + // Cache filenames are exactly 45 chars (.envelope). + if (strlen(filename) <= 45) { + return false; + } + + char *end; + uint64_t ts = strtoull(filename, &end, 10); + if (*end != '-') { + return false; + } + + const char *count_str = end + 1; + long count = strtol(count_str, &end, 10); + if (*end != '-' || count < 0) { + return false; + } + + const char *uuid = end + 1; + size_t tail_len = strlen(uuid); + // 36 chars UUID (with dashes) + ".envelope" + if (tail_len != 36 + 9 || strcmp(uuid + 36, ".envelope") != 0) { + return false; + } + + *ts_out = ts; + *count_out = (int)count; + *uuid_out = uuid; + return true; +} + +sentry_path_t * +sentry__run_make_cache_path( + const sentry_run_t *run, uint64_t ts, int count, const char *uuid) +{ + char filename[128]; + snprintf(filename, sizeof(filename), "%" PRIu64 "-%02d-%.36s.envelope", ts, + count, uuid); + return sentry__path_join_str(run->cache_path, filename); +} + +bool +sentry__run_move_cache( + const sentry_run_t *run, const sentry_path_t *src, int retry_count) +{ + if (sentry__path_create_dir_all(run->cache_path) != 0) { + SENTRY_ERRORF("mkdir failed: \"%s\"", run->cache_path->path); + return false; + } + + char filename[128]; + const char *src_name = sentry__path_filename(src); + uint64_t parsed_ts; + int parsed_count; + const char *parsed_uuid; + const char *cache_name = sentry__parse_cache_filename(src_name, &parsed_ts, + &parsed_count, &parsed_uuid) + ? parsed_uuid + : src_name; + if (retry_count >= 0) { + snprintf(filename, sizeof(filename), "%" PRIu64 "-%02d-%.36s.envelope", + sentry__usec_time() / 1000, retry_count, cache_name); + } else { + snprintf(filename, sizeof(filename), "%s", cache_name); + } + + sentry_path_t *dst_path = sentry__path_join_str(run->cache_path, filename); + if (!dst_path) { + return false; + } + + int rv = sentry__path_rename(src, dst_path); + sentry__path_free(dst_path); + if (rv != 0) { + SENTRY_WARNF("failed to cache envelope \"%s\"", src_name); + return false; + } + + return true; } bool @@ -239,13 +368,9 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) continue; } - sentry_path_t *cache_dir = NULL; - if (options->cache_keep) { - cache_dir = sentry__path_join_str(options->database_path, "cache"); - if (cache_dir) { - sentry__path_create_dir_all(cache_dir); - } - } + bool can_cache = options->cache_keep + && (!options->http_retry + || !sentry__transport_can_retry(options->transport)); sentry_pathiter_t *run_iter = sentry__path_iter_directory(run_dir); const sentry_path_t *file; @@ -292,15 +417,8 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) sentry_envelope_t *envelope = sentry__envelope_from_path(file); sentry__capture_envelope(options->transport, envelope); - if (cache_dir) { - sentry_path_t *cached_file = sentry__path_join_str( - cache_dir, sentry__path_filename(file)); - if (!cached_file - || sentry__path_rename(file, cached_file) != 0) { - SENTRY_WARNF("failed to cache envelope \"%s\"", - sentry__path_filename(file)); - } - sentry__path_free(cached_file); + if (can_cache + && sentry__run_move_cache(options->run, file, -1)) { continue; } } @@ -309,7 +427,6 @@ sentry__process_old_runs(const sentry_options_t *options, uint64_t last_crash) } sentry__pathiter_free(run_iter); - sentry__path_free(cache_dir); sentry__path_remove_all(run_dir); sentry__filelock_free(lock); } diff --git a/src/sentry_database.h b/src/sentry_database.h index 791c30d9f..056752d01 100644 --- a/src/sentry_database.h +++ b/src/sentry_database.h @@ -11,7 +11,9 @@ typedef struct sentry_run_s { sentry_path_t *run_path; sentry_path_t *session_path; sentry_path_t *external_path; + sentry_path_t *cache_path; sentry_filelock_t *lock; + long refcount; } sentry_run_t; /** @@ -22,6 +24,11 @@ typedef struct sentry_run_s { */ sentry_run_t *sentry__run_new(const sentry_path_t *database_path); +/** + * Increment the refcount and return the run pointer. + */ +sentry_run_t *sentry__run_incref(sentry_run_t *run); + /** * This will clean up all the files belonging to this run. */ @@ -63,6 +70,28 @@ bool sentry__run_write_session( */ bool sentry__run_clear_session(const sentry_run_t *run); +/** + * This will serialize and write the given envelope to disk into the cache + * directory. When retry_count >= 0 the filename uses retry format + * `--.envelope`, otherwise `.envelope`. + */ +bool sentry__run_write_cache(const sentry_run_t *run, + const sentry_envelope_t *envelope, int retry_count); + +/** + * Moves a file into the cache directory. When retry_count >= 0 the + * destination uses retry format `--.envelope`, + * otherwise the original filename is preserved. + */ +bool sentry__run_move_cache( + const sentry_run_t *run, const sentry_path_t *src, int retry_count); + +/** + * Builds a cache path: `/cache/--.envelope`. + */ +sentry_path_t *sentry__run_make_cache_path( + const sentry_run_t *run, uint64_t ts, int count, const char *uuid); + /** * This function is essential to send crash reports from previous runs of the * program. @@ -78,6 +107,13 @@ bool sentry__run_clear_session(const sentry_run_t *run); void sentry__process_old_runs( const sentry_options_t *options, uint64_t last_crash); +/** + * Parses a retry cache filename: `--.envelope`. + * Returns false for plain cache filenames (`.envelope`). + */ +bool sentry__parse_cache_filename(const char *filename, uint64_t *ts_out, + int *count_out, const char **uuid_out); + /** * Cleans up the cache based on options.max_cache_size and * options.max_cache_age. diff --git a/src/sentry_options.c b/src/sentry_options.c index dcbffff92..8cc2eb587 100644 --- a/src/sentry_options.c +++ b/src/sentry_options.c @@ -78,6 +78,7 @@ sentry_options_new(void) opts, sentry__getenv_double("SENTRY_TRACES_SAMPLE_RATE", 0.0)); opts->max_spans = SENTRY_SPANS_MAX; opts->handler_strategy = SENTRY_HANDLER_STRATEGY_DEFAULT; + opts->http_retry = true; return opts; } @@ -839,6 +840,18 @@ sentry_options_set_handler_strategy( #endif // SENTRY_PLATFORM_LINUX +void +sentry_options_set_http_retry(sentry_options_t *opts, int enabled) +{ + opts->http_retry = !!enabled; +} + +int +sentry_options_get_http_retry(const sentry_options_t *opts) +{ + return opts->http_retry; +} + void sentry_options_set_propagate_traceparent( sentry_options_t *opts, int propagate_traceparent) diff --git a/src/sentry_options.h b/src/sentry_options.h index ca186389b..9cda3d00a 100644 --- a/src/sentry_options.h +++ b/src/sentry_options.h @@ -78,6 +78,7 @@ struct sentry_options_s { bool enable_metrics; sentry_before_send_metric_function_t before_send_metric_func; void *before_send_metric_data; + bool http_retry; /* everything from here on down are options which are stored here but not exposed through the options API */ diff --git a/src/sentry_retry.c b/src/sentry_retry.c new file mode 100644 index 000000000..6e2509a34 --- /dev/null +++ b/src/sentry_retry.c @@ -0,0 +1,363 @@ +#include "sentry_retry.h" +#include "sentry_alloc.h" +#include "sentry_database.h" +#include "sentry_envelope.h" +#include "sentry_logger.h" +#include "sentry_options.h" +#include "sentry_utils.h" + +#include +#include + +#define SENTRY_RETRY_ATTEMPTS 6 +#define SENTRY_RETRY_INTERVAL (15 * 60 * 1000) +#define SENTRY_RETRY_THROTTLE 100 + +typedef enum { + SENTRY_RETRY_STARTUP = 0, + SENTRY_RETRY_RUNNING = 1, + SENTRY_RETRY_SEALED = 2 +} sentry_retry_state_t; + +typedef enum { + SENTRY_POLL_IDLE = 0, + SENTRY_POLL_SCHEDULED = 1, + SENTRY_POLL_SHUTDOWN = 2 +} sentry_poll_state_t; + +struct sentry_retry_s { + sentry_run_t *run; + bool cache_keep; + uint64_t startup_time; + volatile long state; + volatile long scheduled; + sentry_bgworker_t *bgworker; + sentry_retry_send_func_t send_cb; + void *send_data; + sentry_mutex_t sealed_lock; + uintptr_t sealed_envelope; +}; + +sentry_retry_t * +sentry__retry_new(const sentry_options_t *options) +{ + sentry_retry_t *retry = SENTRY_MAKE(sentry_retry_t); + if (!retry) { + return NULL; + } + memset(retry, 0, sizeof(sentry_retry_t)); + sentry__mutex_init(&retry->sealed_lock); + retry->run = sentry__run_incref(options->run); + retry->cache_keep = options->cache_keep; + retry->startup_time = sentry__usec_time() / 1000; + return retry; +} + +void +sentry__retry_free(sentry_retry_t *retry) +{ + if (!retry) { + return; + } + sentry__mutex_free(&retry->sealed_lock); + sentry__run_free(retry->run); + sentry_free(retry); +} + +uint64_t +sentry__retry_backoff(int count) +{ + return (uint64_t)SENTRY_RETRY_INTERVAL << MIN(MAX(count, 0), 5); +} + +typedef struct { + sentry_path_t *path; + uint64_t ts; + int count; + char uuid[37]; +} retry_item_t; + +static int +compare_retry_items(const void *a, const void *b) +{ + const retry_item_t *ia = a; + const retry_item_t *ib = b; + if (ia->ts != ib->ts) { + return ia->ts < ib->ts ? -1 : 1; + } + if (ia->count != ib->count) { + return ia->count - ib->count; + } + return strcmp(ia->uuid, ib->uuid); +} + +static bool +handle_result(sentry_retry_t *retry, const retry_item_t *item, int status_code) +{ + // Only network failures (status_code < 0) trigger retries. HTTP responses + // including 5xx (500, 502, 503, 504) are discarded: + // https://develop.sentry.dev/sdk/expected-features/#dealing-with-network-failures + + // network failure with retries remaining: bump count & re-enqueue + if (item->count + 1 < SENTRY_RETRY_ATTEMPTS && status_code < 0) { + sentry_path_t *new_path = sentry__run_make_cache_path(retry->run, + sentry__usec_time() / 1000, item->count + 1, item->uuid); + if (new_path) { + if (sentry__path_rename(item->path, new_path) != 0) { + SENTRY_WARNF( + "failed to rename retry envelope \"%s\"", item->path->path); + } + sentry__path_free(new_path); + } + return true; + } + + bool exhausted = item->count + 1 >= SENTRY_RETRY_ATTEMPTS; + + // network failure with retries exhausted + if (exhausted && status_code < 0) { + if (retry->cache_keep) { + SENTRY_WARNF("max retries (%d) reached, moving envelope to cache", + SENTRY_RETRY_ATTEMPTS); + } else { + SENTRY_WARNF("max retries (%d) reached, discarding envelope", + SENTRY_RETRY_ATTEMPTS); + } + } + + // cache on last attempt + if (exhausted && retry->cache_keep) { + if (!sentry__run_move_cache(retry->run, item->path, -1)) { + sentry__path_remove(item->path); + } + return false; + } + + sentry__path_remove(item->path); + return false; +} + +size_t +sentry__retry_send(sentry_retry_t *retry, uint64_t before, + sentry_retry_send_func_t send_cb, void *data) +{ + sentry_pathiter_t *piter + = sentry__path_iter_directory(retry->run->cache_path); + if (!piter) { + return 0; + } + + size_t item_cap = 16; + retry_item_t *items = sentry_malloc(item_cap * sizeof(retry_item_t)); + if (!items) { + sentry__pathiter_free(piter); + return 0; + } + + size_t total = 0; + size_t eligible = 0; + uint64_t now = before > 0 ? 0 : sentry__usec_time() / 1000; + + const sentry_path_t *p; + while ((p = sentry__pathiter_next(piter)) != NULL) { + const char *fname = sentry__path_filename(p); + uint64_t ts; + int count; + const char *uuid; + if (!sentry__parse_cache_filename(fname, &ts, &count, &uuid)) { + continue; + } + if (before > 0 && ts >= before) { + continue; + } + total++; + if (!before + && (now < ts || (now - ts) < sentry__retry_backoff(count))) { + continue; + } + if (eligible == item_cap) { + item_cap *= 2; + retry_item_t *tmp = sentry_malloc(item_cap * sizeof(retry_item_t)); + if (!tmp) { + break; + } + memcpy(tmp, items, eligible * sizeof(retry_item_t)); + sentry_free(items); + items = tmp; + } + retry_item_t *item = &items[eligible]; + item->path = sentry__path_clone(p); + if (!item->path) { + break; + } + item->ts = ts; + item->count = count; + memcpy(item->uuid, uuid, 36); + item->uuid[36] = '\0'; + eligible++; + } + sentry__pathiter_free(piter); + + if (eligible > 1) { + qsort(items, eligible, sizeof(retry_item_t), compare_retry_items); + } + + for (size_t i = 0; i < eligible; i++) { + sentry_envelope_t *envelope = sentry__envelope_from_path(items[i].path); + if (!envelope) { + sentry__path_remove(items[i].path); + total--; + } else { + SENTRY_DEBUGF("retrying envelope (%d/%d)", items[i].count + 1, + SENTRY_RETRY_ATTEMPTS); + int status_code = send_cb(envelope, data); + sentry_envelope_free(envelope); + if (!handle_result(retry, &items[i], status_code)) { + total--; + } + // stop on network failure to avoid wasting time on a dead + // connection; remaining envelopes stay untouched for later + if (status_code < 0) { + break; + } + } + } + + for (size_t i = 0; i < eligible; i++) { + sentry__path_free(items[i].path); + } + sentry_free(items); + return total; +} + +static void +retry_poll_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + uint64_t before + = sentry__atomic_fetch(&retry->state) == SENTRY_RETRY_STARTUP + ? retry->startup_time + : 0; + // CAS instead of unconditional store to preserve SENTRY_POLL_SHUTDOWN + sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_SCHEDULED, SENTRY_POLL_IDLE); + if (sentry__retry_send(retry, before, retry->send_cb, retry->send_data) + && sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_IDLE, SENTRY_POLL_SCHEDULED)) { + sentry__bgworker_submit_delayed(retry->bgworker, retry_poll_task, NULL, + retry, SENTRY_RETRY_INTERVAL); + } + // subsequent polls use backoff instead of the startup time filter + sentry__atomic_compare_swap( + &retry->state, SENTRY_RETRY_STARTUP, SENTRY_RETRY_RUNNING); +} + +void +sentry__retry_start(sentry_retry_t *retry, sentry_bgworker_t *bgworker, + sentry_retry_send_func_t send_cb, void *send_data) +{ + retry->bgworker = bgworker; + retry->send_cb = send_cb; + retry->send_data = send_data; + sentry__atomic_store(&retry->scheduled, SENTRY_POLL_SCHEDULED); + sentry__bgworker_submit_delayed( + bgworker, retry_poll_task, NULL, retry, SENTRY_RETRY_THROTTLE); +} + +static void +retry_flush_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + if (sentry__atomic_compare_swap( + &retry->state, SENTRY_RETRY_STARTUP, SENTRY_RETRY_RUNNING)) { + sentry__retry_send(retry, UINT64_MAX, retry->send_cb, retry->send_data); + } +} + +static bool +drop_task_cb(void *_data, void *_ctx) +{ + (void)_data; + (void)_ctx; + return true; +} + +void +sentry__retry_shutdown(sentry_retry_t *retry) +{ + if (retry) { + // drop the delayed poll and prevent retry_poll_task from re-arming + sentry__bgworker_foreach_matching( + retry->bgworker, retry_poll_task, drop_task_cb, NULL); + sentry__atomic_store(&retry->scheduled, SENTRY_POLL_SHUTDOWN); + sentry__bgworker_submit(retry->bgworker, retry_flush_task, NULL, retry); + } +} + +static bool +retry_dump_cb(void *_envelope, void *_retry) +{ + sentry_retry_t *retry = (sentry_retry_t *)_retry; + sentry_envelope_t *envelope = (sentry_envelope_t *)_envelope; + if ((uintptr_t)envelope != retry->sealed_envelope) { + sentry__run_write_cache(retry->run, envelope, 0); + } else { + retry->sealed_envelope = 0; + } + return true; +} + +void +sentry__retry_dump_queue( + sentry_retry_t *retry, sentry_task_exec_func_t task_func) +{ + if (retry) { + // prevent duplicate writes from a still-running detached worker + sentry__mutex_lock(&retry->sealed_lock); + sentry__atomic_store(&retry->state, SENTRY_RETRY_SEALED); + sentry__mutex_unlock(&retry->sealed_lock); + + sentry__bgworker_foreach_matching( + retry->bgworker, task_func, retry_dump_cb, retry); + } +} + +static void +retry_trigger_task(void *_retry, void *_state) +{ + (void)_state; + sentry_retry_t *retry = _retry; + sentry__retry_send(retry, UINT64_MAX, retry->send_cb, retry->send_data); +} + +void +sentry__retry_trigger(sentry_retry_t *retry) +{ + sentry__bgworker_submit(retry->bgworker, retry_trigger_task, NULL, retry); +} + +void +sentry__retry_enqueue(sentry_retry_t *retry, const sentry_envelope_t *envelope) +{ + sentry__mutex_lock(&retry->sealed_lock); + if (sentry__atomic_fetch(&retry->state) == SENTRY_RETRY_SEALED) { + sentry__mutex_unlock(&retry->sealed_lock); + return; + } + if (!sentry__run_write_cache(retry->run, envelope, 0)) { + sentry__mutex_unlock(&retry->sealed_lock); + return; + } + retry->sealed_envelope = (uintptr_t)envelope; + sentry__mutex_unlock(&retry->sealed_lock); + + sentry__atomic_compare_swap( + &retry->state, SENTRY_RETRY_STARTUP, SENTRY_RETRY_RUNNING); + if (sentry__atomic_compare_swap( + &retry->scheduled, SENTRY_POLL_IDLE, SENTRY_POLL_SCHEDULED)) { + sentry__bgworker_submit_delayed(retry->bgworker, retry_poll_task, NULL, + retry, SENTRY_RETRY_INTERVAL); + } +} diff --git a/src/sentry_retry.h b/src/sentry_retry.h new file mode 100644 index 000000000..5e5222097 --- /dev/null +++ b/src/sentry_retry.h @@ -0,0 +1,57 @@ +#ifndef SENTRY_RETRY_H_INCLUDED +#define SENTRY_RETRY_H_INCLUDED + +#include "sentry_boot.h" +#include "sentry_path.h" +#include "sentry_sync.h" + +typedef struct sentry_retry_s sentry_retry_t; + +typedef int (*sentry_retry_send_func_t)( + sentry_envelope_t *envelope, void *data); + +sentry_retry_t *sentry__retry_new(const sentry_options_t *options); +void sentry__retry_free(sentry_retry_t *retry); + +/** + * Schedules retry polling on `bgworker` using `send_cb`. + */ +void sentry__retry_start(sentry_retry_t *retry, sentry_bgworker_t *bgworker, + sentry_retry_send_func_t send_cb, void *send_data); + +/** + * Prepares retry for shutdown: drops pending polls and submits a flush task. + */ +void sentry__retry_shutdown(sentry_retry_t *retry); + +/** + * Dumps queued envelopes to the retry dir and seals against further writes. + */ +void sentry__retry_dump_queue( + sentry_retry_t *retry, sentry_task_exec_func_t task_func); + +/** + * Writes a failed envelope to the retry dir and schedules a delayed poll. + */ +void sentry__retry_enqueue( + sentry_retry_t *retry, const sentry_envelope_t *envelope); + +/** + * Sends eligible retry files via `send_cb`. `before > 0`: send files with + * ts < before (startup). `before == 0`: use backoff. Returns remaining file + * count for controlling polling. + */ +size_t sentry__retry_send(sentry_retry_t *retry, uint64_t before, + sentry_retry_send_func_t send_cb, void *data); + +/** + * Exponential backoff: 15m, 30m, 1h, 2h, 4h, 8h, 8h, ... (capped at 8h). + */ +uint64_t sentry__retry_backoff(int count); + +/** + * Submits a delayed retry poll task on the background worker. + */ +void sentry__retry_trigger(sentry_retry_t *retry); + +#endif diff --git a/src/sentry_sync.c b/src/sentry_sync.c index c0d903f03..8e9067b99 100644 --- a/src/sentry_sync.c +++ b/src/sentry_sync.c @@ -425,7 +425,8 @@ shutdown_task(void *task_data, void *UNUSED(state)) } int -sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout) +sentry__bgworker_shutdown_cb(sentry_bgworker_t *bgw, uint64_t timeout, + void (*on_timeout)(void *), void *on_timeout_data) { if (!sentry__atomic_fetch(&bgw->running)) { SENTRY_WARN("trying to shut down non-running thread"); @@ -441,12 +442,22 @@ sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout) while (true) { uint64_t now = sentry__monotonic_time(); if (now > started && now - started > timeout) { - sentry__atomic_store(&bgw->running, 0); - sentry__thread_detach(bgw->thread_id); - sentry__mutex_unlock(&bgw->task_lock); - SENTRY_WARN( - "background thread failed to shut down cleanly within timeout"); - return 1; + if (on_timeout) { + // Unblock the worker (e.g. close transport handles) and + // let it finish in-flight work like handle_result. + sentry__mutex_unlock(&bgw->task_lock); + on_timeout(on_timeout_data); + on_timeout = NULL; + sentry__mutex_lock(&bgw->task_lock); + // fall through to !running check below + } else { + sentry__atomic_store(&bgw->running, 0); + sentry__thread_detach(bgw->thread_id); + sentry__mutex_unlock(&bgw->task_lock); + SENTRY_WARN("background thread failed to shut down cleanly " + "within timeout"); + return 1; + } } if (!sentry__atomic_fetch(&bgw->running)) { diff --git a/src/sentry_sync.h b/src/sentry_sync.h index 73f0b8e92..f110a81ca 100644 --- a/src/sentry_sync.h +++ b/src/sentry_sync.h @@ -469,8 +469,20 @@ int sentry__bgworker_flush(sentry_bgworker_t *bgw, uint64_t timeout); /** * This will try to shut down the background worker thread, with a `timeout`. * Returns 0 on success. + * + * The `_cb` variant accepts an `on_timeout` callback that is invoked when + * the timeout expires, just before detaching the thread. This gives the + * caller a chance to unblock the worker (e.g. by closing transport handles) + * so it can finish in-flight work. */ -int sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout); +int sentry__bgworker_shutdown_cb(sentry_bgworker_t *bgw, uint64_t timeout, + void (*on_timeout)(void *), void *on_timeout_data); + +static inline int +sentry__bgworker_shutdown(sentry_bgworker_t *bgw, uint64_t timeout) +{ + return sentry__bgworker_shutdown_cb(bgw, timeout, NULL, NULL); +} /** * This will set a preferable thread name for background worker. diff --git a/src/sentry_transport.c b/src/sentry_transport.c index 6b63c5783..1b81cb652 100644 --- a/src/sentry_transport.c +++ b/src/sentry_transport.c @@ -10,6 +10,7 @@ struct sentry_transport_s { int (*flush_func)(uint64_t timeout, void *state); void (*free_func)(void *state); size_t (*dump_func)(sentry_run_t *run, void *state); + void (*retry_func)(void *state); void *state; bool running; }; @@ -147,3 +148,24 @@ sentry__transport_get_state(sentry_transport_t *transport) { return transport ? transport->state : NULL; } + +void +sentry_transport_retry(sentry_transport_t *transport) +{ + if (transport && transport->retry_func) { + transport->retry_func(transport->state); + } +} + +void +sentry__transport_set_retry_func( + sentry_transport_t *transport, void (*retry_func)(void *state)) +{ + transport->retry_func = retry_func; +} + +bool +sentry__transport_can_retry(sentry_transport_t *transport) +{ + return transport && transport->retry_func; +} diff --git a/src/sentry_transport.h b/src/sentry_transport.h index 036233284..5ed1e7b81 100644 --- a/src/sentry_transport.h +++ b/src/sentry_transport.h @@ -57,4 +57,8 @@ size_t sentry__transport_dump_queue( void *sentry__transport_get_state(sentry_transport_t *transport); +void sentry__transport_set_retry_func( + sentry_transport_t *transport, void (*retry_func)(void *state)); +bool sentry__transport_can_retry(sentry_transport_t *transport); + #endif diff --git a/src/transports/sentry_http_transport.c b/src/transports/sentry_http_transport.c index 24b1ba566..237052689 100644 --- a/src/transports/sentry_http_transport.c +++ b/src/transports/sentry_http_transport.c @@ -4,8 +4,10 @@ #include "sentry_envelope.h" #include "sentry_options.h" #include "sentry_ratelimiter.h" +#include "sentry_retry.h" #include "sentry_string.h" #include "sentry_transport.h" +#include "sentry_utils.h" #ifdef SENTRY_TRANSPORT_COMPRESSION # include "zlib.h" @@ -29,6 +31,7 @@ typedef struct { int (*start_client)(void *, const sentry_options_t *); sentry_http_send_func_t send_func; void (*shutdown_client)(void *client); + sentry_retry_t *retry; } http_transport_state_t; #ifdef SENTRY_TRANSPORT_COMPRESSION @@ -182,6 +185,54 @@ sentry__prepared_http_request_free(sentry_prepared_http_request_t *req) sentry_free(req); } +static int +http_send_request( + http_transport_state_t *state, sentry_prepared_http_request_t *req) +{ + sentry_http_response_t resp; + memset(&resp, 0, sizeof(resp)); + + if (!state->send_func(state->client, req, &resp)) { + sentry_free(resp.retry_after); + sentry_free(resp.x_sentry_rate_limits); + return -1; + } + + if (resp.x_sentry_rate_limits) { + sentry__rate_limiter_update_from_header( + state->ratelimiter, resp.x_sentry_rate_limits); + } else if (resp.retry_after) { + sentry__rate_limiter_update_from_http_retry_after( + state->ratelimiter, resp.retry_after); + } else if (resp.status_code == 429) { + sentry__rate_limiter_update_from_429(state->ratelimiter); + } + + sentry_free(resp.retry_after); + sentry_free(resp.x_sentry_rate_limits); + return resp.status_code; +} + +static int +http_send_envelope(http_transport_state_t *state, sentry_envelope_t *envelope) +{ + sentry_prepared_http_request_t *req = sentry__prepare_http_request( + envelope, state->dsn, state->ratelimiter, state->user_agent); + if (!req) { + return 0; + } + int status_code = http_send_request(state, req); + sentry__prepared_http_request_free(req); + return status_code; +} + +static int +retry_send_cb(sentry_envelope_t *envelope, void *_state) +{ + http_transport_state_t *state = _state; + return http_send_envelope(state, envelope); +} + static void http_transport_state_free(void *_state) { @@ -192,6 +243,7 @@ http_transport_state_free(void *_state) sentry__dsn_decref(state->dsn); sentry_free(state->user_agent); sentry__rate_limiter_free(state->ratelimiter); + sentry__retry_free(state->retry); sentry_free(state); } @@ -201,29 +253,19 @@ http_send_task(void *_envelope, void *_state) sentry_envelope_t *envelope = _envelope; http_transport_state_t *state = _state; - sentry_prepared_http_request_t *req = sentry__prepare_http_request( - envelope, state->dsn, state->ratelimiter, state->user_agent); - if (!req) { - return; + int status_code = http_send_envelope(state, envelope); + if (status_code < 0 && state->retry) { + sentry__retry_enqueue(state->retry, envelope); } +} - sentry_http_response_t resp; - memset(&resp, 0, sizeof(resp)); - - if (state->send_func(state->client, req, &resp)) { - if (resp.x_sentry_rate_limits) { - sentry__rate_limiter_update_from_header( - state->ratelimiter, resp.x_sentry_rate_limits); - } else if (resp.retry_after) { - sentry__rate_limiter_update_from_http_retry_after( - state->ratelimiter, resp.retry_after); - } else if (resp.status_code == 429) { - sentry__rate_limiter_update_from_429(state->ratelimiter); - } +static void +http_transport_shutdown_timeout(void *_state) +{ + http_transport_state_t *state = _state; + if (state->shutdown_client) { + state->shutdown_client(state->client); } - sentry_free(resp.retry_after); - sentry_free(resp.x_sentry_rate_limits); - sentry__prepared_http_request_free(req); } static int @@ -244,7 +286,23 @@ http_transport_start(const sentry_options_t *options, void *transport_state) } } - return sentry__bgworker_start(bgworker); + int rv = sentry__bgworker_start(bgworker); + if (rv != 0) { + return rv; + } + + if (options->http_retry) { + state->retry = sentry__retry_new(options); + if (state->retry) { + sentry__retry_start(state->retry, bgworker, retry_send_cb, state); + } else { + // cannot retry, clear retry_func so envelopes get cached instead of + // dropped + sentry__transport_set_retry_func(options->transport, NULL); + } + } + + return 0; } static int @@ -260,9 +318,12 @@ http_transport_shutdown(uint64_t timeout, void *transport_state) sentry_bgworker_t *bgworker = transport_state; http_transport_state_t *state = sentry__bgworker_get_state(bgworker); - int rv = sentry__bgworker_shutdown(bgworker, timeout); - if (rv != 0 && state->shutdown_client) { - state->shutdown_client(state->client); + sentry__retry_shutdown(state->retry); + + int rv = sentry__bgworker_shutdown_cb( + bgworker, timeout, http_transport_shutdown_timeout, state); + if (rv != 0) { + sentry__retry_dump_queue(state->retry, http_send_task); } return rv; } @@ -298,6 +359,16 @@ http_transport_get_state(sentry_transport_t *transport) return sentry__bgworker_get_state(bgworker); } +static void +http_transport_retry(void *transport_state) +{ + sentry_bgworker_t *bgworker = transport_state; + http_transport_state_t *state = sentry__bgworker_get_state(bgworker); + if (state->retry) { + sentry__retry_trigger(state->retry); + } +} + sentry_transport_t * sentry__http_transport_new(void *client, sentry_http_send_func_t send_func) { @@ -331,6 +402,7 @@ sentry__http_transport_new(void *client, sentry_http_send_func_t send_func) sentry_transport_set_flush_func(transport, http_transport_flush); sentry_transport_set_shutdown_func(transport, http_transport_shutdown); sentry__transport_set_dump_func(transport, http_dump_queue); + sentry__transport_set_retry_func(transport, http_transport_retry); return transport; } diff --git a/src/transports/sentry_http_transport_curl.c b/src/transports/sentry_http_transport_curl.c index b0c967f5c..eec48f8e4 100644 --- a/src/transports/sentry_http_transport_curl.c +++ b/src/transports/sentry_http_transport_curl.c @@ -189,6 +189,7 @@ curl_send_task(void *_client, sentry_prepared_http_request_t *req, curl_easy_setopt(curl, CURLOPT_POSTFIELDS, req->body); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)req->body_len); curl_easy_setopt(curl, CURLOPT_USERAGENT, SENTRY_SDK_USER_AGENT); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 15000L); char error_buf[CURL_ERROR_SIZE]; error_buf[0] = 0; diff --git a/src/transports/sentry_http_transport_winhttp.c b/src/transports/sentry_http_transport_winhttp.c index 0997d3562..f358fa6cb 100644 --- a/src/transports/sentry_http_transport_winhttp.c +++ b/src/transports/sentry_http_transport_winhttp.c @@ -134,6 +134,9 @@ winhttp_client_start(void *_client, const sentry_options_t *opts) return 1; } + // 15s resolve/connect, 30s send/receive (WinHTTP defaults) + WinHttpSetTimeouts(client->session, 15000, 15000, 30000, 30000); + return 0; } @@ -154,9 +157,9 @@ winhttp_client_shutdown(void *_client) WinHttpCloseHandle(client->session); client->session = NULL; } - if (client->request) { - WinHttpCloseHandle(client->request); - client->request = NULL; + HINTERNET request = InterlockedExchangePointer(&client->request, NULL); + if (request) { + WinHttpCloseHandle(request); } } @@ -302,10 +305,9 @@ winhttp_send_task(void *_client, sentry_prepared_http_request_t *req, uint64_t now = sentry__monotonic_time(); SENTRY_DEBUGF("request handled in %llums", now - started); -exit: - if (client->request) { - HINTERNET request = client->request; - client->request = NULL; +exit:; + HINTERNET request = InterlockedExchangePointer(&client->request, NULL); + if (request) { WinHttpCloseHandle(request); } sentry_free(url); diff --git a/tests/test_integration_cache.py b/tests/test_integration_cache.py index aff10fa9f..6ba9fd225 100644 --- a/tests/test_integration_cache.py +++ b/tests/test_integration_cache.py @@ -179,3 +179,57 @@ def test_cache_max_items(cmake, backend): assert cache_dir.exists() cache_files = list(cache_dir.glob("*.envelope")) assert len(cache_files) == 5 + + +@pytest.mark.parametrize( + "backend", + [ + "inproc", + pytest.param( + "breakpad", + marks=pytest.mark.skipif( + not has_breakpad, reason="breakpad backend not available" + ), + ), + ], +) +def test_cache_max_items_with_retry(cmake, backend): + tmp_path = cmake( + ["sentry_example"], {"SENTRY_BACKEND": backend, "SENTRY_TRANSPORT": "none"} + ) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + # Create cache files via crash+restart cycles + for i in range(4): + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "crash"], + expect_failure=True, + ) + + # Move envelopes into cache + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + ) + + # Pre-populate cache/ with retry-format envelope files + cache_dir.mkdir(parents=True, exist_ok=True) + for i in range(4): + ts = int(time.time() * 1000) + f = cache_dir / f"{ts}-00-00000000-0000-0000-0000-{i:012x}.envelope" + f.write_text("dummy envelope content") + + # Trigger sentry_init which runs cleanup + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + ) + + # max 5 items total in cache/ + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) <= 5 diff --git a/tests/test_integration_http.py b/tests/test_integration_http.py index a78d2e322..c85da1702 100644 --- a/tests/test_integration_http.py +++ b/tests/test_integration_http.py @@ -37,6 +37,8 @@ pytestmark = pytest.mark.skipif(not has_http, reason="tests need http") +unreachable_dsn = "http://uiaeosnrtdy@127.0.0.1:19999/123456" + # fmt: off auth_header = ( f"Sentry sentry_key=uiaeosnrtdy, sentry_version=7, sentry_client=sentry.native/{SENTRY_VERSION}" @@ -725,3 +727,344 @@ def test_discarding_before_breadcrumb_http(cmake, httpserver): assert_event(envelope) assert_no_breadcrumbs(envelope) + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_on_network_error(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + # unreachable port triggers CURLE_COULDNT_CONNECT + env_unreachable = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "capture-event"], + env=env_unreachable, + ) + + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + + # retry on next run with working server + env_reachable = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "no-setup"], + env=env_reachable, + ) + assert waiting.result + + assert len(httpserver.log) == 1 + envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + assert envelope.headers["event_id"] == envelope_uuid + assert_meta(envelope, integration="inproc") + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_attempts(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run(tmp_path, "sentry_example", ["log", "capture-event"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + envelope = Envelope.deserialize(cache_files[0].read_bytes()) + assert envelope.headers["event_id"] == envelope_uuid + + run(tmp_path, "sentry_example", ["log", "no-setup"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-01-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + run(tmp_path, "sentry_example", ["log", "no-setup"], env=env) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-02-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + # exhaust remaining retries (max 6) + for i in range(4): + run(tmp_path, "sentry_example", ["log", "no-setup"], env=env) + + # discarded after max retries (cache_keep not enabled) + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_with_cache_keep(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env_unreachable = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "capture-event"], + env=env_unreachable, + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + env_reachable = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env_reachable, + ) + assert waiting.result + + assert len(list(cache_dir.glob("*.envelope"))) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_cache_keep_max_attempts(cmake): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "capture-event"], + env=env, + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + for _ in range(5): + run( + tmp_path, + "sentry_example", + ["log", "cache-keep", "no-setup"], + env=env, + ) + + assert cache_dir.exists() + assert len(list(cache_dir.glob("*.envelope"))) == 1 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_http_error_discards_envelope(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "Internal Server Error", status=500 + ) + + with httpserver.wait(timeout=10) as waiting: + run(tmp_path, "sentry_example", ["log", "capture-event"], env=env) + assert waiting.result + + # HTTP errors discard, not retry + cache_files = list(cache_dir.glob("*.envelope")) if cache_dir.exists() else [] + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_rate_limit_discards_envelope(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "inproc"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + env = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "Rate Limited", status=429, headers={"retry-after": "60"} + ) + + with httpserver.wait(timeout=10) as waiting: + run(tmp_path, "sentry_example", ["log", "capture-event"], env=env) + assert waiting.result + + # 429 discards, not retry + cache_files = list(cache_dir.glob("*.envelope")) if cache_dir.exists() else [] + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_success(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env_unreachable = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "capture-multiple"], + env=env_unreachable, + ) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 10 + + env_reachable = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + for _ in range(10): + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data( + "OK" + ) + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "no-setup"], + env=env_reachable, + ) + assert waiting.result + + assert len(httpserver.log) == 10 + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_network_error(cmake): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "capture-multiple"], + env=env, + ) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 10 + + run( + tmp_path, + "sentry_example", + ["log", "no-setup"], + env=env, + ) + + # first envelope retried and bumped, rest untouched (stop on failure) + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 10 + assert len([f for f in cache_files if "-00-" in f.name]) == 9 + assert len([f for f in cache_files if "-01-" in f.name]) == 1 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_multiple_rate_limit(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env_unreachable = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "capture-multiple"], + env=env_unreachable, + ) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 10 + + # rate limit response followed by discards for the rest (rate limiter + # kicks in after the first 429) + env_reachable = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + httpserver.expect_request("/api/123456/envelope/").respond_with_data( + "Rate Limited", status=429, headers={"retry-after": "60"} + ) + + run( + tmp_path, + "sentry_example", + ["log", "no-setup"], + env=env_reachable, + ) + + # first envelope gets 429, rest are discarded by rate limiter + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 + + +@pytest.mark.skipif(not has_files, reason="test needs a local filesystem") +def test_http_retry_session_on_network_error(cmake, httpserver): + tmp_path = cmake(["sentry_example"], {"SENTRY_BACKEND": "none"}) + cache_dir = tmp_path.joinpath(".sentry-native/cache") + + env_unreachable = dict(os.environ, SENTRY_DSN=unreachable_dsn) + + run( + tmp_path, + "sentry_example", + ["log", "start-session"], + env=env_unreachable, + ) + + assert cache_dir.exists() + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-00-" in str(cache_files[0].name) + envelope_uuid = cache_files[0].stem[-36:] + + # second and third attempts still fail — envelope gets renamed each time + run(tmp_path, "sentry_example", ["log", "no-setup"], env=env_unreachable) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-01-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + run(tmp_path, "sentry_example", ["log", "no-setup"], env=env_unreachable) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 1 + assert "-02-" in str(cache_files[0].name) + assert cache_files[0].stem[-36:] == envelope_uuid + + # succeed on fourth attempt + env_reachable = dict(os.environ, SENTRY_DSN=make_dsn(httpserver)) + httpserver.expect_oneshot_request("/api/123456/envelope/").respond_with_data("OK") + + with httpserver.wait(timeout=10) as waiting: + run( + tmp_path, + "sentry_example", + ["log", "no-setup"], + env=env_reachable, + ) + assert waiting.result + + assert len(httpserver.log) == 1 + envelope = Envelope.deserialize(httpserver.log[0][0].get_data()) + assert_session(envelope, {"init": True, "status": "exited", "errors": 0}) + + cache_files = list(cache_dir.glob("*.envelope")) + assert len(cache_files) == 0 diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 359ec43f9..a3b88efa5 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -42,6 +42,7 @@ add_executable(sentry_test_unit test_path.c test_process.c test_ratelimiter.c + test_retry.c test_ringbuffer.c test_sampling.c test_scope.c diff --git a/tests/unit/test_cache.c b/tests/unit/test_cache.c index e161340bc..f49637bc4 100644 --- a/tests/unit/test_cache.c +++ b/tests/unit/test_cache.c @@ -46,6 +46,7 @@ SENTRY_TEST(cache_keep) SENTRY_TEST_OPTIONS_NEW(options); sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); sentry_options_set_cache_keep(options, true); + sentry_options_set_http_retry(options, false); sentry_init(options); sentry_path_t *cache_path @@ -243,6 +244,69 @@ SENTRY_TEST(cache_max_items) sentry_close(); } +SENTRY_TEST(cache_max_items_with_retry) +{ +#if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) + SKIP_TEST(); +#endif + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_cache_keep(options, true); + sentry_options_set_cache_max_items(options, 7); + sentry_init(options); + + sentry_path_t *cache_path + = sentry__path_join_str(options->database_path, "cache"); + TEST_ASSERT(!!cache_path); + TEST_ASSERT(sentry__path_remove_all(cache_path) == 0); + TEST_ASSERT(sentry__path_create_dir_all(cache_path) == 0); + + time_t now = time(NULL); + + // 5 cache-format files: 1,3,5,7,9 min old + for (int i = 0; i < 5; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char *filename = sentry__uuid_as_filename(&event_id, ".envelope"); + TEST_ASSERT(!!filename); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + sentry_free(filename); + + TEST_ASSERT(sentry__path_touch(filepath) == 0); + TEST_ASSERT(set_file_mtime(filepath, now - ((i * 2 + 1) * 60)) == 0); + sentry__path_free(filepath); + } + + // 5 retry-format files: 0,2,4,6,8 min old + for (int i = 0; i < 5; i++) { + sentry_uuid_t event_id = sentry_uuid_new_v4(); + char uuid[37]; + sentry_uuid_as_string(&event_id, uuid); + char filename[128]; + snprintf(filename, sizeof(filename), "%" PRIu64 "-00-%.36s.envelope", + (uint64_t)now, uuid); + sentry_path_t *filepath = sentry__path_join_str(cache_path, filename); + + TEST_ASSERT(sentry__path_touch(filepath) == 0); + TEST_ASSERT(set_file_mtime(filepath, now - (i * 2 * 60)) == 0); + sentry__path_free(filepath); + } + + sentry__cleanup_cache(options); + + int total_count = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(cache_path); + const sentry_path_t *entry; + while (iter && (entry = sentry__pathiter_next(iter)) != NULL) { + total_count++; + } + sentry__pathiter_free(iter); + + TEST_CHECK_INT_EQUAL(total_count, 7); + + sentry__path_free(cache_path); + sentry_close(); +} + SENTRY_TEST(cache_max_size_and_age) { #if defined(SENTRY_PLATFORM_NX) || defined(SENTRY_PLATFORM_PS) diff --git a/tests/unit/test_retry.c b/tests/unit/test_retry.c new file mode 100644 index 000000000..d707dd162 --- /dev/null +++ b/tests/unit/test_retry.c @@ -0,0 +1,462 @@ +#include "sentry_core.h" +#include "sentry_database.h" +#include "sentry_envelope.h" +#include "sentry_options.h" +#include "sentry_path.h" +#include "sentry_retry.h" +#include "sentry_session.h" +#include "sentry_testsupport.h" +#include "sentry_transport.h" +#include "sentry_utils.h" +#include "sentry_uuid.h" + +#include + +static int +count_envelope_files(const sentry_path_t *dir) +{ + int count = 0; + sentry_pathiter_t *iter = sentry__path_iter_directory(dir); + const sentry_path_t *file; + while (iter && (file = sentry__pathiter_next(iter)) != NULL) { + if (sentry__path_ends_with(file, ".envelope")) { + count++; + } + } + sentry__pathiter_free(iter); + return count; +} + +static int +find_envelope_attempt(const sentry_path_t *dir) +{ + sentry_pathiter_t *iter = sentry__path_iter_directory(dir); + const sentry_path_t *file; + while (iter && (file = sentry__pathiter_next(iter)) != NULL) { + if (!sentry__path_ends_with(file, ".envelope")) { + continue; + } + const char *name = sentry__path_filename(file); + uint64_t ts; + int attempt; + const char *uuid; + if (sentry__parse_cache_filename(name, &ts, &attempt, &uuid)) { + sentry__pathiter_free(iter); + return attempt; + } + } + sentry__pathiter_free(iter); + return -1; +} + +static void +write_retry_file(const sentry_run_t *run, uint64_t timestamp, int retry_count, + const sentry_uuid_t *event_id) +{ + sentry_envelope_t *envelope = sentry__envelope_new(); + sentry_value_t event = sentry__value_new_event_with_id(event_id); + sentry__envelope_add_event(envelope, event); + + char uuid[37]; + sentry_uuid_as_string(event_id, uuid); + + sentry_path_t *path + = sentry__run_make_cache_path(run, timestamp, retry_count, uuid); + (void)sentry_envelope_write_to_path(envelope, path); + sentry__path_free(path); + sentry_envelope_free(envelope); +} + +typedef struct { + int status_code; + size_t count; +} retry_test_ctx_t; + +static int +test_send_cb(sentry_envelope_t *envelope, void *_ctx) +{ + (void)envelope; + retry_test_ctx_t *ctx = _ctx; + ctx->count++; + return ctx->status_code; +} + +SENTRY_TEST(retry_filename) +{ + uint64_t ts; + int count; + const char *uuid; + + TEST_CHECK(sentry__parse_cache_filename( + "1234567890-00-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, + &count, &uuid)); + TEST_CHECK_UINT64_EQUAL(ts, 1234567890); + TEST_CHECK_INT_EQUAL(count, 0); + TEST_CHECK(strncmp(uuid, "abcdefab-1234-5678-9abc-def012345678", 36) == 0); + + TEST_CHECK(sentry__parse_cache_filename( + "999-04-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, + &uuid)); + TEST_CHECK_UINT64_EQUAL(ts, 999); + TEST_CHECK_INT_EQUAL(count, 4); + + // negative count + TEST_CHECK(!sentry__parse_cache_filename( + "123--01-abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, + &uuid)); + + // cache filename (no timestamp/count) + TEST_CHECK(!sentry__parse_cache_filename( + "abcdefab-1234-5678-9abc-def012345678.envelope", &ts, &count, &uuid)); + + // missing .envelope suffix + TEST_CHECK(!sentry__parse_cache_filename( + "123-00-abcdefab-1234-5678-9abc-def012345678.txt", &ts, &count, &uuid)); +} + +SENTRY_TEST(retry_throttle) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t ids[4]; + for (int i = 0; i < 4; i++) { + ids[i] = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 0, &ids[i]); + } + + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 4); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 4); + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_skew) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + // future timestamp simulates clock moving backward + uint64_t future_ts = sentry__usec_time() / 1000 + 1000000; + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, future_ts, 0, &event_id); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + + // item should NOT be processed due to backoff (clock backward) + TEST_CHECK_INT_EQUAL(ctx.count, 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_result) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + + // 1. Success (200) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 0); + + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 2. Rate limited (429) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + ctx = (retry_test_ctx_t) { 429, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 3. Discard (0) → removes + write_retry_file(options->run, old_ts, 0, &event_id); + ctx = (retry_test_ctx_t) { 0, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // 4. Network error → bumps count + write_retry_file(options->run, old_ts, 0, &event_id); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 0); + + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 1); + + // 5. Network error at last attempt → removed + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + uint64_t very_old_ts + = sentry__usec_time() / 1000 - 2 * sentry__retry_backoff(5); + write_retry_file(options->run, very_old_ts, 5, &event_id); + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_session) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_release(options, "test@1.0.0"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + sentry__path_remove_all(options->run->cache_path); + sentry__path_create_dir_all(options->run->cache_path); + + sentry_session_t *session = sentry__session_new(); + TEST_ASSERT(!!session); + sentry_envelope_t *envelope = sentry__envelope_new(); + TEST_ASSERT(!!envelope); + sentry__envelope_add_session(envelope, session); + + TEST_CHECK(sentry__run_write_cache(options->run, envelope, 0)); + TEST_CHECK_INT_EQUAL(count_envelope_files(options->run->cache_path), 1); + + sentry_envelope_free(envelope); + sentry__session_free(session); + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_cache) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_options_set_cache_keep(options, 1); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts = sentry__usec_time() / 1000 - 2 * sentry__retry_backoff(5); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 5, &event_id); + + char uuid_str[37]; + sentry_uuid_as_string(&event_id, uuid_str); + char cache_name[46]; + snprintf(cache_name, sizeof(cache_name), "%.36s.envelope", uuid_str); + sentry_path_t *cached = sentry__path_join_str(cache_path, cache_name); + + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK(!sentry__path_is_file(cached)); + + // Network error on a file at count=5 with max_retries=6 → renames to + // cache format (.envelope) + retry_test_ctx_t ctx = { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK(sentry__path_is_file(cached)); + + // Success on a file at count=5 → also renames to cache format + // (cache_keep preserves all envelopes regardless of send outcome) + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + write_retry_file(options->run, old_ts, 5, &event_id); + TEST_CHECK(!sentry__path_is_file(cached)); + + ctx = (retry_test_ctx_t) { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 1); + TEST_CHECK(sentry__path_is_file(cached)); + + sentry__retry_free(retry); + sentry__path_free(cached); + sentry_close(); +} + +static int retry_func_calls = 0; + +static void +mock_retry_func(void *state) +{ + (void)state; + retry_func_calls++; +} + +static void +noop_send(sentry_envelope_t *envelope, void *state) +{ + (void)state; + sentry_envelope_free(envelope); +} + +SENTRY_TEST(transport_retry) +{ + // no retry_func → no-op + sentry_transport_t *transport = sentry_transport_new(noop_send); + TEST_CHECK(!sentry__transport_can_retry(transport)); + sentry_transport_retry(transport); + + // with retry_func → calls it + retry_func_calls = 0; + sentry__transport_set_retry_func(transport, mock_retry_func); + TEST_CHECK(sentry__transport_can_retry(transport)); + sentry_transport_retry(transport); + TEST_CHECK_INT_EQUAL(retry_func_calls, 1); + + // NULL transport → no-op + sentry_transport_retry(NULL); + TEST_CHECK_INT_EQUAL(retry_func_calls, 1); + + sentry_transport_free(transport); +} + +SENTRY_TEST(retry_backoff) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t base = sentry__retry_backoff(0); + uint64_t ref = sentry__usec_time() / 1000 - 10 * base; + + // retry 0: 10*base old, eligible (backoff=base) + sentry_uuid_t id1 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref, 0, &id1); + + // retry 1: 1*base old, not yet eligible (backoff=2*base) + sentry_uuid_t id2 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref + 9 * base, 1, &id2); + + // retry 1: 10*base old, eligible (backoff=2*base) + sentry_uuid_t id3 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref, 1, &id3); + + // retry 2: 2*base old, not eligible (backoff=4*base) + sentry_uuid_t id4 = sentry_uuid_new_v4(); + write_retry_file(options->run, ref + 8 * base, 2, &id4); + + // With backoff: only eligible ones (id1 and id3) are sent + retry_test_ctx_t ctx = { 200, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 2); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 2); + + // Startup scan (no backoff check): remaining 2 files are sent + ctx = (retry_test_ctx_t) { 200, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 2); + TEST_CHECK_INT_EQUAL(count_envelope_files(cache_path), 0); + + // Verify backoff calculation + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(0), base); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(1), base * 2); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(2), base * 4); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(3), base * 8); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(4), base * 16); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(5), base * 32); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(6), base * 32); + TEST_CHECK_UINT64_EQUAL(sentry__retry_backoff(-1), base); + + sentry__retry_free(retry); + sentry_close(); +} + +SENTRY_TEST(retry_trigger) +{ + SENTRY_TEST_OPTIONS_NEW(options); + sentry_options_set_dsn(options, "https://foo@sentry.invalid/42"); + sentry_options_set_http_retry(options, false); + sentry_init(options); + + sentry_retry_t *retry = sentry__retry_new(options); + TEST_ASSERT(!!retry); + + const sentry_path_t *cache_path = options->run->cache_path; + sentry__path_remove_all(cache_path); + sentry__path_create_dir_all(cache_path); + + uint64_t old_ts + = sentry__usec_time() / 1000 - 10 * sentry__retry_backoff(0); + sentry_uuid_t event_id = sentry_uuid_new_v4(); + write_retry_file(options->run, old_ts, 0, &event_id); + + // UINT64_MAX (trigger mode) bypasses backoff: bumps count + retry_test_ctx_t ctx = { -1, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 1); + + // second call: bumps again because UINT64_MAX skips backoff + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, UINT64_MAX, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 1); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 2); + + // before=0 (poll mode) respects backoff: item is skipped + ctx = (retry_test_ctx_t) { -1, 0 }; + sentry__retry_send(retry, 0, test_send_cb, &ctx); + TEST_CHECK_INT_EQUAL(ctx.count, 0); + TEST_CHECK_INT_EQUAL(find_envelope_attempt(cache_path), 2); + + sentry__retry_free(retry); + sentry_close(); +} diff --git a/tests/unit/tests.inc b/tests/unit/tests.inc index 60f751b20..a5de45549 100644 --- a/tests/unit/tests.inc +++ b/tests/unit/tests.inc @@ -43,6 +43,7 @@ XX(build_id_parser) XX(cache_keep) XX(cache_max_age) XX(cache_max_items) +XX(cache_max_items_with_retry) XX(cache_max_size) XX(cache_max_size_and_age) XX(capture_minidump_basic) @@ -167,6 +168,14 @@ XX(read_envelope_from_file) XX(read_write_envelope_to_file_null) XX(read_write_envelope_to_invalid_path) XX(recursive_paths) +XX(retry_backoff) +XX(retry_cache) +XX(retry_filename) +XX(retry_result) +XX(retry_session) +XX(retry_skew) +XX(retry_throttle) +XX(retry_trigger) XX(ringbuffer_append) XX(ringbuffer_append_invalid_decref_value) XX(ringbuffer_append_null_decref_value) @@ -215,6 +224,7 @@ XX(traceparent_header_disabled_by_default) XX(traceparent_header_generation) XX(transaction_name_backfill_on_finish) XX(transactions_skip_before_send) +XX(transport_retry) XX(transport_sampling_transactions) XX(transport_sampling_transactions_set_trace) XX(txn_data)