diff --git a/plugins/sql.c b/plugins/sql.c index cd0a52200d9f..f065fc588e45 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -162,6 +162,7 @@ struct sql { int gosstore_fd ; size_t gosstore_nodes_off, gosstore_channels_off; u64 next_rowid; + u32 limit_per_list; /* This is an aux_command for all our watches */ struct command *waitcmd; @@ -1715,14 +1716,13 @@ static const char *db_table_name(const tal_t *ctx, const char *cmdname) return ret; } -#define LIMIT_PER_LIST 10000 - static struct command_result *limited_list_done(struct command *cmd, const char *method, const char *buf, const jsmntok_t *result, struct db_query *dbq) { + struct sql *sql = sql_of(dbq->cmd->plugin); struct table_desc *td = dbq->tables[0]; struct command_result *ret; size_t num_entries; @@ -1735,7 +1735,7 @@ static struct command_result *limited_list_done(struct command *cmd, return ret; /* If we got the number we asked for, we need to ask again. */ - return one_refresh_done(cmd, dbq, num_entries == LIMIT_PER_LIST); + return one_refresh_done(cmd, dbq, num_entries == sql->limit_per_list); } /* The simplest case: append-only lists */ @@ -1743,18 +1743,18 @@ static struct command_result *refresh_by_created_index(struct command *cmd, struct table_desc *td, struct db_query *dbq) { + struct sql *sql = sql_of(dbq->cmd->plugin); struct out_req *req; - /* Since we're relying on watches, mark refreshing unnecessary to start */ - assert(td->refresh_needs != REFRESH_UNNECESSARY); - td->refresh_needs = REFRESH_UNNECESSARY; + /* We no longer need refresh_created, but wait could update this meanwhile. */ + td->refresh_needs &= ~REFRESH_CREATED; req = jsonrpc_request_start(cmd, td->cmdname, limited_list_done, forward_error, dbq); json_add_string(req->js, "index", "created"); json_add_u64(req->js, "start", td->last_created_index + 1); - json_add_u64(req->js, "limit", LIMIT_PER_LIST); + json_add_u64(req->js, "limit", sql->limit_per_list); return send_outreq(req); } @@ -1781,7 +1781,6 @@ static struct command_result *updated_list_done(struct command *cmd, return refresh_by_created_index(cmd, td, dbq); } - td->refresh_needs = REFRESH_UNNECESSARY; return one_refresh_done(cmd, dbq, false); } @@ -1793,6 +1792,7 @@ static struct command_result *paginated_refresh(struct command *cmd, * entire thing */ if (td->refresh_needs & REFRESH_DELETED) { plugin_log(cmd->plugin, LOG_DBG, "%s: total reload due to delete", td->name); + /* Since this reloads everything, covers all: updates and creates */ td->refresh_needs = REFRESH_UNNECESSARY; return default_refresh(cmd, td, dbq); } @@ -1801,6 +1801,7 @@ static struct command_result *paginated_refresh(struct command *cmd, struct out_req *req; plugin_log(cmd->plugin, LOG_DBG, "%s: records updated, updating from %"PRIu64, td->name, td->last_updated_index + 1); + td->refresh_needs &= ~REFRESH_UPDATED; req = jsonrpc_request_start(cmd, td->cmdname, updated_list_done, forward_error, dbq); @@ -2216,11 +2217,16 @@ int main(int argc, char *argv[]) sql->gosstore_fd = -1; sql->gosstore_nodes_off = sql->gosstore_channels_off = 0; sql->next_rowid = 1; + sql->limit_per_list = 10000; plugin_main(argv, init, take(sql), PLUGIN_RESTARTABLE, true, NULL, commands, ARRAY_SIZE(commands), NULL, 0, NULL, 0, NULL, 0, plugin_option_dev("dev-sqlfilename", "string", "Use on-disk sqlite3 file instead of in memory (e.g. debugging)", charp_option, NULL, &sql->dbfilename), + plugin_option_dev("dev-sqllistlimit", + "int", + "A variable setting for how many chainmoves/channelmoves we fetch at once", + u32_option, NULL, &sql->limit_per_list), NULL); } diff --git a/tests/test_plugin.py b/tests/test_plugin.py index df6193db96ab..3f4f11cbe90f 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -4410,6 +4410,16 @@ def test_sql_deprecated(node_factory, bitcoind): assert ret == {'rows': [[1]]} +def test_sql_limit_per_list(node_factory): + l1, l2, l3 = node_factory.line_graph( + 3, wait_for_announce=True, opts=[{}, {"dev-sqllistlimit": 10}, {}] + ) + for i in range(20): + inv = l3.rpc.invoice(1000, f"inv-{i}", f"inv-{i}")["bolt11"] + l1.rpc.xpay(inv) + l2.rpc.sql("SELECT created_index, payment_hash FROM channelmoves") + + def test_plugin_persist_option(node_factory): """test that options from config file get remembered across plugin stop/start""" plugin_path = os.path.join(os.getcwd(), 'contrib/plugins/helloworld.py')