bookkeeper: restore limit on asking for all channelmoves at once.

Now we've found all the issues, the latency spike (4 seconds on my laptop)
for querying 2M elements remains.

Restore the limited sampling which we reverted, but make it 10,000 now.

This doesn't help our worst-case latency, because sql still asks for all 2M entries on
first access.  We address that next.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

plugins/bkpr/bookkeeper.c

@@ -54,6 +54,10 @@ static struct refresh_info *use_rinfo(struct refresh_info *rinfo)
 	return rinfo;
 }
 
+/* Recursion */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo);
+
 static struct command_result *rinfo_one_done(struct command *cmd,
 					     struct refresh_info *rinfo)
 {
@@ -104,6 +108,7 @@ static struct fee_sum *find_sum_for_txid(struct fee_sum **sums,
 	return NULL;
 }
 
+#define LISTCHANNELMOVES_LIMIT 10000
 static struct command_result *listchannelmoves_done(struct command *cmd,
 						    const char *method,
 						    const char *buf,
@@ -125,16 +130,36 @@ static struct command_result *listchannelmoves_done(struct command *cmd,
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
 
+	/* If there might be more, try asking for more */
+	if (moves->size == LISTCHANNELMOVES_LIMIT)
+		limited_listchannelmoves(cmd, rinfo);
+
 	return rinfo_one_done(cmd, rinfo);
 }
 
+/* We do 1000 at a time to avoid overwhelming lightningd */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo)
+{
+	struct bkpr *bkpr = bkpr_of(cmd->plugin);
+	struct out_req *req;
+
+	req = jsonrpc_request_start(cmd, "listchannelmoves",
+				    listchannelmoves_done,
+				    plugin_broken_cb,
+				    use_rinfo(rinfo));
+	json_add_string(req->js, "index", "created");
+	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
+	json_add_u64(req->js, "limit", LISTCHANNELMOVES_LIMIT);
+	return send_outreq(req);
+}
+
 static struct command_result *listchainmoves_done(struct command *cmd,
 						  const char *method,
 						  const char *buf,
 						  const jsmntok_t *result,
 						  struct refresh_info *rinfo)
 {
-	struct out_req *req;
 	const jsmntok_t *moves, *t;
 	size_t i;
 	struct bkpr *bkpr = bkpr_of(cmd->plugin);
@@ -150,13 +175,7 @@ static struct command_result *listchainmoves_done(struct command *cmd,
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
 
-	req = jsonrpc_request_start(cmd, "listchannelmoves",
-				    listchannelmoves_done,
-				    plugin_broken_cb,
-				    use_rinfo(rinfo));
-	json_add_string(req->js, "index", "created");
-	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
-	send_outreq(req);
+	limited_listchannelmoves(cmd, rinfo);
 	return rinfo_one_done(cmd, rinfo);
 }