feat(renderd): configure request queue limits

Komzpa · Komzpa · commit c3c4bea969ef · 2026-05-04T01:49:36.000+04:00
diff --git a/docs/man/renderd.conf.5 b/docs/man/renderd.conf.5
@@ -44,6 +44,19 @@ Specify the number of threads to be used for \fBrenderd\fR.
 A value of \fB'-1'\fR will configure \fBnum_threads\fR to the number of cores on the system.
 The default value is \fB'4'\fR (macro definition \fB'NUM_THREADS'\fR).
 
+.TP
+.B request_queue_limit
+Specify the maximum number of requests kept in each time-critical request queue: priority, normal, low priority, and bulk.
+When one of these queues is full, new requests overflow into the dirty queue if space is available.
+The default value is \fB'256'\fR when metatiles are enabled, or \fB'512'\fR without metatiles (macro definition \fB'DEFAULT_REQUEST_QUEUE_LIMIT'\fR).
+
+.TP
+.B dirty_queue_limit
+Specify the maximum number of requests kept in the dirty queue.
+The dirty queue is the background queue for expired or overflowed tiles where no client is waiting for the response.
+Larger values can preserve more expired work for quiet periods, but they can also increase memory use and make it easier for important overflowed tiles to sit behind lower-priority background work.
+The default value is \fB'8000'\fR when metatiles are enabled, or \fB'10000'\fR without metatiles (macro definition \fB'DEFAULT_DIRTY_QUEUE_LIMIT'\fR).
+
 .TP
 .B pid_file
 Specify the file path into which the PID will be written by \fBrenderd\fR.
@@ -189,6 +202,8 @@ Only used by \fBrenderd\fR.
 
 .SH SEE ALSO
 .BR renderd(1)
+.PP
+See \fBdocs/renderd-queue-tuning.md\fR in the source tree for queue tuning and rollout guidance.
 .BR
 
 .SH AUTHOR
diff --git a/docs/renderd-queue-tuning.md b/docs/renderd-queue-tuning.md
@@ -0,0 +1,68 @@
+# renderd Queue Tuning
+
+`renderd` keeps incoming render requests in five queues:
+
+- priority request queue: missing tiles where a client is waiting
+- request queue: stale tiles where a client is waiting
+- low priority request queue: less urgent stale/style refresh work
+- dirty queue: background work where no client waits for the response
+- bulk request queue: explicit bulk rendering work
+
+Requests are fetched in that priority order. When a time-critical request queue
+is full, new requests overflow into the dirty queue. That prevents immediate
+drops, but it also means the overflowed request no longer gets client-waiting
+priority.
+
+## Configuration
+
+Set the queue limits in the active `[renderd]` section of `renderd.conf`:
+
+```ini
+[renderd]
+num_threads=4
+request_queue_limit=256
+dirty_queue_limit=8000
+```
+
+`request_queue_limit` applies separately to each of the priority, normal, low
+priority, and bulk queues. `dirty_queue_limit` applies to the background dirty
+queue. If the dirty queue is also full, new overflow work is dropped.
+
+The defaults keep the previous compiled-in behavior:
+
+- with metatiles: `request_queue_limit=256`, `dirty_queue_limit=8000`
+- without metatiles: `request_queue_limit=512`, `dirty_queue_limit=10000`
+
+## Operating Guidance
+
+Increase `dirty_queue_limit` when dirty tiles are dropped during peak load but
+the server has quiet periods later where it can catch up. This is useful for
+large public tile services where preserving expired work can improve freshness
+after the peak passes.
+
+Do not increase `dirty_queue_limit` blindly on an overloaded server. If the
+higher-priority queues are continuously non-empty, dirty work may still starve,
+and a larger dirty queue mainly stores more backlog. Watch queue length, queue
+time, render throughput, and dropped-tile metrics before and after the change.
+
+Increase gradually. For example, move from the default to a limit sized for
+roughly 10 minutes of dirty work, then one hour, before trying day-scale values.
+The right value depends on metatile size, render throughput, expiry volume, and
+whether the service has predictable low-load windows.
+
+## Rollout Checklist
+
+1. Record current `renderd_queue`, `renderd_queue_time`, and
+   `renderd_processed` Munin graphs.
+2. Set `dirty_queue_limit` in `renderd.conf`.
+3. Restart `renderd` during a maintenance window.
+4. Confirm startup logs show the intended `request_queue_limit` and
+   `dirty_queue_limit`.
+5. Watch whether dropped dirty work decreases without sustained growth in queue
+   time.
+6. Roll back to the previous values if dirty queue time grows continuously or
+   missing-tile requests appear to be delayed by old background work.
+
+This setting only changes queue capacity. It does not implement bounded
+overtaking or fairness between queues; those would require a larger queue
+scheduler change.
diff --git a/etc/renderd/renderd.conf b/etc/renderd/renderd.conf
@@ -6,6 +6,8 @@ stats_file=/run/renderd/renderd.stats
 socketname=/run/renderd/renderd.sock
 num_threads=4
 tile_dir=/var/cache/renderd/tiles
+;request_queue_limit=256
+;dirty_queue_limit=8000
 
 [mapnik]
 plugins_dir=/usr/lib/mapnik/3.1/input
diff --git a/etc/renderd/renderd.conf.examples b/etc/renderd/renderd.conf.examples
@@ -6,12 +6,16 @@ stats_file=/run/renderd/renderd.stats
 socketname=/run/renderd/renderd.sock
 num_threads=4
 tile_dir=/var/cache/renderd/tiles
+;request_queue_limit=256
+;dirty_queue_limit=8000
 
 ;[renderd]
 ;iphostname=::1
 ;ipport=7654
 ;num_threads=4
 ;tile_dir=rados://tiles/etc/ceph/ceph.conf
+;request_queue_limit=256
+;dirty_queue_limit=8000
 ;pid_file=/run/renderd/renderd_rados.pid
 ;stats_file=/run/renderd/renderd.stats
 
diff --git a/etc/renderd/renderd.conf.in b/etc/renderd/renderd.conf.in
@@ -3,6 +3,8 @@ pid_file=@RENDERD_PIDFILE@
 socketname=@RENDERD_SOCKET@
 stats_file=@RENDERD_RUN_DIR@/renderd.stats
 tile_dir=@RENDERD_TILE_DIR@
+; request_queue_limit=256
+; dirty_queue_limit=8000
 
 [mapnik]
 font_dir=@MAPNIK_FONTS_DIR@
diff --git a/includes/render_config.h b/includes/render_config.h
@@ -114,14 +114,13 @@
 // Metatiles are much larger in size so we don't need big queues to handle large areas
 #ifdef METATILE
 #define QUEUE_MAX (64)
-#define REQ_LIMIT (256)
-#define DIRTY_LIMIT (8000)
+#define DEFAULT_REQUEST_QUEUE_LIMIT (256)
+#define DEFAULT_DIRTY_QUEUE_LIMIT (8000)
 
 #else
 #define QUEUE_MAX (1024)
-#define REQ_LIMIT (512)
-#define DIRTY_LIMIT (10000)
-#define HASHIDX_SIZE 22123
+#define DEFAULT_REQUEST_QUEUE_LIMIT (512)
+#define DEFAULT_DIRTY_QUEUE_LIMIT (10000)
 #endif
 
 // Penalty for client making an invalid request (in seconds)
diff --git a/includes/renderd.h b/includes/renderd.h
@@ -42,9 +42,11 @@ typedef struct {
 	const char *socketname;
 	const char *stats_filename;
 	const char *tile_dir;
+	int dirty_queue_limit;
 	int ipport;
 	int mapnik_font_dir_recurse;
 	int num_threads;
+	int request_queue_limit;
 } renderd_config;
 
 typedef struct {
diff --git a/includes/request_queue.h b/includes/request_queue.h
@@ -50,7 +50,9 @@ struct item_idx {
 };
 
 struct request_queue {
+	int dirtyLimit;
 	int hashidxSize;
+	int requestLimit;
 	struct item reqHead, reqPrioHead, reqLowHead, reqBulkHead, dirtyHead, renderHead;
 	struct item_idx *item_hashidx;
 	int reqNum, reqPrioNum, reqLowNum, reqBulkNum, dirtyNum;
@@ -60,6 +62,7 @@ struct request_queue {
 };
 
 struct request_queue *request_queue_init();
+struct request_queue *request_queue_init_with_limits(int request_limit, int dirty_limit);
 void request_queue_close(struct request_queue *queue);
 
 struct item *request_queue_fetch_request(struct request_queue *queue);
diff --git a/src/renderd.c b/src/renderd.c
@@ -809,7 +809,7 @@ int main(int argc, char **argv)
 	}
 
 	g_logger(G_LOG_LEVEL_INFO, "Initialising request queue");
-	render_request_queue = request_queue_init();
+	render_request_queue = request_queue_init_with_limits(config.request_queue_limit, config.dirty_queue_limit);
 
 	if (render_request_queue == NULL) {
 		g_logger(G_LOG_LEVEL_CRITICAL, "Failed to initialise request queue");
diff --git a/src/renderd_config.c b/src/renderd_config.c
@@ -425,7 +425,9 @@ void process_renderd_sections(dictionary *ini, const char *config_file_name, ren
 			copy_string(section, &configs_dest[renderd_section_num].name, renderd_strlen + 2);
 
 			process_config_int(ini, section, "ipport", &configs_dest[renderd_section_num].ipport, 0);
+			process_config_int(ini, section, "dirty_queue_limit", &configs_dest[renderd_section_num].dirty_queue_limit, DEFAULT_DIRTY_QUEUE_LIMIT);
 			process_config_int(ini, section, "num_threads", &configs_dest[renderd_section_num].num_threads, NUM_THREADS);
+			process_config_int(ini, section, "request_queue_limit", &configs_dest[renderd_section_num].request_queue_limit, DEFAULT_REQUEST_QUEUE_LIMIT);
 			process_config_string(ini, section, "iphostname", &configs_dest[renderd_section_num].iphostname, "", INILINE_MAX);
 			process_config_string(ini, section, "pid_file", &configs_dest[renderd_section_num].pid_filename, RENDERD_PIDFILE, PATH_MAX);
 			process_config_string(ini, section, "socketname", &configs_dest[renderd_section_num].socketname, RENDERD_SOCKET, PATH_MAX);
@@ -436,6 +438,16 @@ void process_renderd_sections(dictionary *ini, const char *config_file_name, ren
 				configs_dest[renderd_section_num].num_threads = sysconf(_SC_NPROCESSORS_ONLN);
 			}
 
+			if (configs_dest[renderd_section_num].request_queue_limit < 1) {
+				g_logger(G_LOG_LEVEL_CRITICAL, "Specified request_queue_limit (%i) is too small, must be greater than or equal to %i.", configs_dest[renderd_section_num].request_queue_limit, 1);
+				exit(7);
+			}
+
+			if (configs_dest[renderd_section_num].dirty_queue_limit < 0) {
+				g_logger(G_LOG_LEVEL_CRITICAL, "Specified dirty_queue_limit (%i) is too small, must be greater than or equal to %i.", configs_dest[renderd_section_num].dirty_queue_limit, 0);
+				exit(7);
+			}
+
 			if (strnlen(configs_dest[renderd_section_num].socketname, PATH_MAX) >= renderd_socketname_maxlen) {
 				g_logger(G_LOG_LEVEL_CRITICAL, "Specified socketname (%s) exceeds maximum allowed length of %i.", configs_dest[renderd_section_num].socketname, renderd_socketname_maxlen);
 				exit(7);
@@ -510,6 +522,8 @@ void process_config_file(const char *config_file_name, int active_renderd_sectio
 		}
 
 		g_logger(G_LOG_LEVEL_DEBUG, "\trenderd(%i): num_threads = '%i'", i, config_slaves[i].num_threads);
+		g_logger(G_LOG_LEVEL_DEBUG, "\trenderd(%i): request_queue_limit = '%i'", i, config_slaves[i].request_queue_limit);
+		g_logger(G_LOG_LEVEL_DEBUG, "\trenderd(%i): dirty_queue_limit = '%i'", i, config_slaves[i].dirty_queue_limit);
 		g_logger(G_LOG_LEVEL_DEBUG, "\trenderd(%i): pid_file = '%s'", i, config_slaves[i].pid_filename);
 
 		if (strnlen(config_slaves[i].stats_filename, PATH_MAX)) {
@@ -526,6 +540,8 @@ void process_config_file(const char *config_file_name, int active_renderd_sectio
 	}
 
 	g_logger(log_level, "\trenderd: num_threads = '%i'", config.num_threads);
+	g_logger(log_level, "\trenderd: request_queue_limit = '%i'", config.request_queue_limit);
+	g_logger(log_level, "\trenderd: dirty_queue_limit = '%i'", config.dirty_queue_limit);
 
 	if (active_renderd_section_num == 0 && num_slave_threads > 0) {
 		g_logger(log_level, "\trenderd: num_slave_threads = '%i'", num_slave_threads);
diff --git a/src/request_queue.c b/src/request_queue.c
@@ -21,11 +21,30 @@
 #include <unistd.h>
 #include <string.h>
 #include <pthread.h>
+#include <limits.h>
 
 #include "render_config.h"
 #include "request_queue.h"
 #include "g_logger.h"
 
+#define MIN_HASHIDX_SIZE 2213
+
+static int request_queue_hash_size(int request_limit, int dirty_limit)
+{
+	/*
+	 * Keep the de-duplication index roughly proportional to the configured
+	 * queue capacity. Large dirty queues are useful only if duplicate lookup
+	 * remains cheap while the queue is backlogged.
+	 */
+	size_t queue_capacity = (4 * (size_t) request_limit) + (size_t) dirty_limit;
+
+	if (queue_capacity > INT_MAX) {
+		return INT_MAX;
+	}
+
+	return MAX(MIN_HASHIDX_SIZE, (int) queue_capacity);
+}
+
 static int calcHashKey(struct request_queue *queue, struct item *item)
 {
 	uint64_t xmlnameHash = 0;
@@ -306,27 +325,27 @@ enum protoCmd request_queue_add_request(struct request_queue * queue, struct ite
 	}
 
 	// New request, add it to render or dirty queue
-	if ((req->cmd == cmdRender) && (queue->reqNum < REQ_LIMIT)) {
+	if ((req->cmd == cmdRender) && (queue->reqNum < queue->requestLimit)) {
 		list = &(queue->reqHead);
 		item->inQueue = queueRequest;
 		item->originatedQueue = queueRequest;
 		queue->reqNum++;
-	} else if ((req->cmd == cmdRenderPrio) && (queue->reqPrioNum < REQ_LIMIT)) {
+	} else if ((req->cmd == cmdRenderPrio) && (queue->reqPrioNum < queue->requestLimit)) {
 		list = &(queue->reqPrioHead);
 		item->inQueue = queueRequestPrio;
 		item->originatedQueue = queueRequestPrio;
 		queue->reqPrioNum++;
-	} else if ((req->cmd == cmdRenderLow) && (queue->reqLowNum < REQ_LIMIT)) {
+	} else if ((req->cmd == cmdRenderLow) && (queue->reqLowNum < queue->requestLimit)) {
 		list = &(queue->reqLowHead);
 		item->inQueue = queueRequestLow;
 		item->originatedQueue = queueRequestLow;
 		queue->reqLowNum++;
-	} else if ((req->cmd == cmdRenderBulk) && (queue->reqBulkNum < REQ_LIMIT)) {
+	} else if ((req->cmd == cmdRenderBulk) && (queue->reqBulkNum < queue->requestLimit)) {
 		list = &(queue->reqBulkHead);
 		item->inQueue = queueRequestBulk;
 		item->originatedQueue = queueRequestBulk;
 		queue->reqBulkNum++;
-	} else if (queue->dirtyNum < DIRTY_LIMIT) {
+	} else if (queue->dirtyNum < queue->dirtyLimit) {
 		list = &(queue->dirtyHead);
 		item->inQueue = queueDirty;
 		item->originatedQueue = queueDirty;
@@ -450,7 +469,7 @@ void request_queue_copy_stats(struct request_queue * queue, stats_struct * stats
 	pthread_mutex_unlock(&queue->qLock);
 }
 
-struct request_queue * request_queue_init()
+struct request_queue * request_queue_init_with_limits(int request_limit, int dirty_limit)
 {
 	int res;
 	struct request_queue * queue = calloc(1, sizeof(struct request_queue));
@@ -459,6 +478,15 @@ struct request_queue * request_queue_init()
 		return NULL;
 	}
 
+	if (request_limit < 1 || dirty_limit < 0) {
+		g_logger(G_LOG_LEVEL_ERROR, "Invalid request queue limits: request_queue_limit=%i dirty_queue_limit=%i", request_limit, dirty_limit);
+		free(queue);
+		return NULL;
+	}
+
+	queue->requestLimit = request_limit;
+	queue->dirtyLimit = dirty_limit;
+
 	res = pthread_mutex_init(&(queue->qLock), NULL);
 
 	if (res != 0) {
@@ -489,13 +517,24 @@ struct request_queue * request_queue_init()
 	queue->reqBulkHead.next = queue->reqBulkHead.prev = &(queue->reqBulkHead);
 	queue->dirtyHead.next = queue->dirtyHead.prev = &(queue->dirtyHead);
 	queue->renderHead.next = queue->renderHead.prev = &(queue->renderHead);
-	queue->hashidxSize = HASHIDX_SIZE;
+	queue->hashidxSize = request_queue_hash_size(request_limit, dirty_limit);
 	queue->item_hashidx = (struct item_idx *) malloc(sizeof(struct item_idx) * queue->hashidxSize);
+	if (queue->item_hashidx == NULL) {
+		g_logger(G_LOG_LEVEL_ERROR, "Failed to initialise request queue index with %i entries", queue->hashidxSize);
+		pthread_mutex_destroy(&(queue->qLock));
+		free(queue);
+		return NULL;
+	}
 	bzero(queue->item_hashidx, sizeof(struct item_idx) * queue->hashidxSize);
 
 	return queue;
 }
 
+struct request_queue * request_queue_init()
+{
+	return request_queue_init_with_limits(DEFAULT_REQUEST_QUEUE_LIMIT, DEFAULT_DIRTY_QUEUE_LIMIT);
+}
+
 void request_queue_close(struct request_queue * queue)
 {
 	//TODO: Free items if the queues are not empty at closing time
diff --git a/tests/gen_tile_test.cpp b/tests/gen_tile_test.cpp
diff --git a/tests/renderd_config_test.cpp b/tests/renderd_config_test.cpp
diff --git a/tests/renderd_config_test_helper.cpp b/tests/renderd_config_test_helper.cpp

Original file line number	Diff line number	Diff line change
`@@ -809,7 +809,7 @@ int main(int argc, char **argv)`
`809`	`809`	`}`
`810`	`810`
`811`	`811`	`g_logger(G_LOG_LEVEL_INFO, "Initialising request queue");`
`812`		`- render_request_queue = request_queue_init();`
	`812`	`+ render_request_queue = request_queue_init_with_limits(config.request_queue_limit, config.dirty_queue_limit);`
`813`	`813`
`814`	`814`	`if (render_request_queue == NULL) {`
`815`	`815`	`g_logger(G_LOG_LEVEL_CRITICAL, "Failed to initialise request queue");`