MEDIUM: lb_fwrr: Use one ebtree per thread group.

When using the round-robin load balancer, the major source of contention
is the lbprm lock, that has to be held every time we pick a server.
To mitigate that, make it so there are one tree per thread-group, and
one lock per thread-group. That means we now have a lb_fwrr_per_tgrp
structure that will contain the two lb_fwrr_groups (active and backup) as well
as the lock to protect them in the per-thread lbprm struct, and all
fields in the struct server are now moved to the per-thread structure
too.
Those changes are mostly mechanical, and brings good performances
improvment, on a 64-cores AMD CPU, with 64 servers configured, we could
process about 620000 requests par second, and we now can process around
1400000 requests per second.

Author: Olivier Houchard

include/haproxy/backend-t.h

@@ -147,6 +147,7 @@
 /* LB parameters for all algorithms, with one instance per thread-group */
 struct lbprm_per_tgrp {
 	union {
+		struct lb_fwrr_per_tgrp fwrr;
 	};
 };
 /* LB parameters for all algorithms */

include/haproxy/lb_fwrr-t.h

@@ -23,6 +23,7 @@
 #define _HAPROXY_LB_FWRR_T_H
 
 #include <import/ebtree-t.h>
+#include <haproxy/thread-t.h>
 
 /* This structure is used to apply fast weighted round robin on a server group */
 struct fwrr_group {
@@ -34,9 +35,13 @@ struct fwrr_group {
 	int curr_weight;        /* total weight of the current time range */
 };
 
-struct lb_fwrr {
+struct lb_fwrr_per_tgrp {
 	struct fwrr_group act;	/* weighted round robin on the active servers */
 	struct fwrr_group bck;	/* weighted round robin on the backup servers */
+	__decl_thread(HA_RWLOCK_T lock);
+};
+
+struct lb_fwrr {
 	int next_weight_act;    /* total weight of the next time range on active servers, for all trees */
 	int next_weight_bck;    /* total weight of the next time range on backup servers, for all trees */
 };

include/haproxy/server-t.h

@@ -275,10 +275,15 @@ struct srv_per_thread {
 struct srv_per_tgroup {
 	struct queue queue;			/* pending connections */
 	struct server *server;                  /* pointer to the corresponding server */
+	struct eb32_node lb_node;               /* node used for tree-based load balancing */
+	struct server *next_full;               /* next server in the temporary full list */
 	unsigned int last_other_tgrp_served;	/* Last other tgrp we dequeued from */
 	unsigned int self_served;		/* Number of connection we dequeued from our own queue */
 	unsigned int dequeuing;                 /* non-zero = dequeuing in progress (atomic) */
 	unsigned int next_takeover;             /* thread ID to try to steal connections from next time */
+	struct eb_root *lb_tree;                 /* For LB algos with split between thread groups, the tree to be used, for each group */
+	unsigned npos, lpos;			/* next and last positions in the LB tree, protected by LB lock */
+	unsigned rweight;			/* remainder of weight in the current LB tree */
 } THREAD_ALIGNED(64);
 
 /* Configure the protocol selection for websocket */
@@ -348,7 +353,6 @@ struct server {
 	unsigned iweight,uweight, cur_eweight;	/* initial weight, user-specified weight, and effective weight */
 	unsigned wscore;			/* weight score, used during srv map computation */
 	unsigned next_eweight;			/* next pending eweight to commit */
-	unsigned rweight;			/* remainder of weight in the current LB tree */
 	unsigned cumulative_weight;		/* weight of servers prior to this one in the same group, for chash balancing */
 	int maxqueue;				/* maximum number of pending connections allowed */
 	unsigned int queueslength;		/* Sum of the length of each queue */
@@ -381,7 +385,6 @@ struct server {
 	 */
 	THREAD_PAD(63);
 	__decl_thread(HA_SPINLOCK_T lock);      /* may enclose the proxy's lock, must not be taken under */
-	unsigned npos, lpos;			/* next and last positions in the LB tree, protected by LB lock */
 	union {
 		struct eb32_node lb_node;       /* node used for tree-based load balancing */
 		struct list lb_list;            /* elem used for list-based load balancing */
@@ -392,7 +395,6 @@ struct server {
 			int lb_lock;                    /* make sure we are the only one updating the server */
 		};
 	};
-	struct server *next_full;               /* next server in the temporary full list */
 
 	/* usually atomically updated by any thread during parsing or on end of request */
 	THREAD_PAD(63);