sql: store short_channel_id as integer for efficient indexing

Short channel IDs were stored as TEXT strings (e.g., "735095x480x1") in
SQLite, which prevented efficient use of indexes on SCID columns.  Change
storage to INTEGER (the u64 encoding), using a custom "SCID" column type
so the result-reading code can detect these columns and format them back
as "NNNxNNNxNNN" strings for backward-compatible JSON output.

Add two new SQL functions:
- scid('NNNxNNNxNNN') -> integer: for efficient WHERE clause filtering
- fmt_scid(integer) -> 'NNNxNNNxNNN': for formatting in SQL expressions

Fixes #8941

Author: vincenzopalazzo

contrib/msggen/msggen/schema.json

@@ -26381,7 +26381,8 @@
                           "INTEGER",
                           "BLOB",
                           "TEXT",
-                          "REAL"
+                          "REAL",
+                          "SCID"
                         ],
                         "description": [
                           "The SQL type of the column."
@@ -26501,15 +26502,15 @@
                   },
                   {
                     "name": "short_channel_id",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "alias_local",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "alias_remote",
-                    "type": "TEXT"
+                    "type": "SCID"
                   },
                   {
                     "name": "opener",
@@ -34093,9 +34094,11 @@
         "  * JSON: string",
         "  * sqlite3: TEXT",
         "",
-        "* *short_channel_id*. A short-channel-id of form 1x2x3.",
+        "* *short_channel_id*. A short-channel-id of form 1x2x3.  Stored as an integer internally for efficient indexing.",
         "  * JSON: string",
-        "  * sqlite3: TEXT"
+        "  * sqlite3: SCID (INTEGER affinity)",
+        "",
+        "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`.  The `fmt_scid()` function converts back to string form."
       ],
       "permitted_sqlite3_functions": [
         "Writing to the database is not permitted, and limits are placed on various other query parameters.",
@@ -34124,7 +34127,9 @@
         "* total",
         "* unixepoch",
         "* json_object",
-        "* json_group_array"
+        "* json_group_array",
+        "* scid",
+        "* fmt_scid"
       ],
       "tables": [
         "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh.  This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).",

doc/schemas/listsqlschemas.json

@@ -68,7 +68,8 @@
                       "INTEGER",
                       "BLOB",
                       "TEXT",
-                      "REAL"
+                      "REAL",
+                      "SCID"
                     ],
                     "description": [
                       "The SQL type of the column."
@@ -188,15 +189,15 @@
               },
               {
                 "name": "short_channel_id",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "alias_local",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "alias_remote",
-                "type": "TEXT"
+                "type": "SCID"
               },
               {
                 "name": "opener",

doc/schemas/sql-template.json

@@ -78,9 +78,11 @@
     "  * JSON: string",
     "  * sqlite3: TEXT",
     "",
-    "* *short_channel_id*. A short-channel-id of form 1x2x3.",
+    "* *short_channel_id*. A short-channel-id of form 1x2x3.  Stored as an integer internally for efficient indexing.",
     "  * JSON: string",
-    "  * sqlite3: TEXT"
+    "  * sqlite3: SCID (INTEGER affinity)",
+    "",
+    "You can use the `scid()` function to convert a short_channel_id string to its integer representation for queries, e.g. `WHERE in_channel = scid('1x2x3')`.  The `fmt_scid()` function converts back to string form."
   ],
   "permitted_sqlite3_functions": [
     "Writing to the database is not permitted, and limits are placed on various other query parameters.",
@@ -109,7 +111,9 @@
     "* total",
     "* unixepoch",
     "* json_object",
-    "* json_group_array"
+    "* json_group_array",
+    "* scid",
+    "* fmt_scid"
   ],
   "tables": [
     "Note that tables which have a `created_index` field use that as the primary key (and `rowid` is an alias to this), otherwise an explicit `rowid` integer primary key is generated, whose value changes on each refresh.  This field is used for related tables to refer to specific rows in their parent. (sqlite3 usually has this as an implicit column, but we make it explicit as the implicit version is not allowed to be used as a foreign key).",

plugins/sql.c

@@ -71,7 +71,7 @@ static const struct fieldtypemap fieldtypemap[] = {
 	{ "boolean", "INTEGER" }, /* FIELD_BOOL */
 	{ "number", "REAL" }, /* FIELD_NUMBER */
 	{ "string", "TEXT" }, /* FIELD_STRING */
-	{ "short_channel_id", "TEXT" }, /* FIELD_SCID */
+	{ "short_channel_id", "SCID" }, /* FIELD_SCID */
 	{ "outpoint", "TEXT" }, /* FIELD_OUTPOINT */
 };
 
@@ -227,6 +227,52 @@ static enum fieldtype find_fieldtype(const jsmntok_t *name)
 	     name->end - name->start, schemas + name->start);
 }
 
+/* SQLite custom function: scid('NNNxNNNxNNN') -> u64 integer.
+ * Allows efficient queries like: WHERE in_channel = scid('735095x480x1') */
+static void sql_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv)
+{
+	struct short_channel_id scid;
+	const char *str;
+
+	if (argc != 1) {
+		sqlite3_result_error(ctx, "scid() requires exactly one argument", -1);
+		return;
+	}
+	if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+		sqlite3_result_null(ctx);
+		return;
+	}
+
+	str = (const char *)sqlite3_value_text(argv[0]);
+	if (!str || !short_channel_id_from_str(str, strlen(str), &scid)) {
+		sqlite3_result_error(ctx, "invalid short_channel_id format, expected NNNxNNNxNNN", -1);
+		return;
+	}
+
+	sqlite3_result_int64(ctx, scid.u64);
+}
+
+/* SQLite custom function: fmt_scid(u64) -> 'NNNxNNNxNNN' string.
+ * Useful for displaying integer SCIDs in text format within SQL expressions. */
+static void sql_fmt_scid_func(sqlite3_context *ctx, int argc, sqlite3_value **argv)
+{
+	struct short_channel_id scid;
+	char *str;
+
+	if (argc != 1) {
+		sqlite3_result_error(ctx, "fmt_scid() requires exactly one argument", -1);
+		return;
+	}
+	if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+		sqlite3_result_null(ctx);
+		return;
+	}
+
+	scid.u64 = sqlite3_value_int64(argv[0]);
+	str = fmt_short_channel_id(tmpctx, scid);
+	sqlite3_result_text(ctx, str, -1, SQLITE_TRANSIENT);
+}
+
 static struct sqlite3 *sqlite_setup(struct plugin *plugin)
 {
 	int err;
@@ -283,6 +329,12 @@ static struct sqlite3 *sqlite_setup(struct plugin *plugin)
 			plugin_err(plugin, "Could not disable sync: %s", errmsg);
 	}
 
+	/* Register custom SCID functions for integer<->text conversion */
+	sqlite3_create_function(db, "scid", 1, SQLITE_UTF8, NULL,
+				sql_scid_func, NULL, NULL);
+	sqlite3_create_function(db, "fmt_scid", 1, SQLITE_UTF8, NULL,
+				sql_fmt_scid_func, NULL, NULL);
+
 	return db;
 }
 
@@ -406,6 +458,10 @@ static int sqlite_authorize(void *dbq_, int code,
 			return SQLITE_OK;
 		if (streq(b, "json_group_array"))
 			return SQLITE_OK;
+		if (streq(b, "scid"))
+			return SQLITE_OK;
+		if (streq(b, "fmt_scid"))
+			return SQLITE_OK;
 	}
 
 	/* See https://www.sqlite.org/c3ref/c_alter_table.html to decode these! */
@@ -447,7 +503,15 @@ static struct command_result *refresh_complete(struct command *cmd,
 			switch (sqlite3_column_type(dbq->stmt, i)) {
 			case SQLITE_INTEGER: {
 				s64 v = sqlite3_column_int64(dbq->stmt, i);
-				json_add_s64(ret, NULL, v);
+				const char *decltype = sqlite3_column_decltype(dbq->stmt, i);
+				if (decltype && streq(decltype, "SCID")) {
+					struct short_channel_id scid;
+					scid.u64 = (u64)v;
+					json_add_string(ret, NULL,
+							fmt_short_channel_id(tmpctx, scid));
+				} else {
+					json_add_s64(ret, NULL, v);
+				}
 				break;
 			}
 			case SQLITE_FLOAT: {
@@ -781,7 +845,18 @@ static struct command_result *process_json_obj(struct command *cmd,
 				}
 				sqlite3_bind_int64(stmt, (*sqloff)++, valmsat.millisatoshis /* Raw: db */);
 				break;
-			case FIELD_SCID:
+			case FIELD_SCID: {
+				struct short_channel_id scid;
+				if (!json_to_short_channel_id(buf, coltok, &scid)) {
+					return command_fail(cmd, LIGHTNINGD,
+							    "column %zu row %zu not a valid short_channel_id: %.*s",
+							    i, row,
+							    json_tok_full_len(coltok),
+							    json_tok_full(buf, coltok));
+				}
+				sqlite3_bind_int64(stmt, (*sqloff)++, scid.u64);
+				break;
+			}
 			case FIELD_STRING:
 			case FIELD_OUTPOINT:
 				sqlite3_bind_text(stmt, (*sqloff)++, buf + coltok->start,
@@ -976,8 +1051,8 @@ static void delete_channel_from_db(struct command *cmd,
 	err = sqlite3_exec(sql->db,
 			   tal_fmt(tmpctx,
 				   "DELETE FROM channels"
-				   " WHERE short_channel_id = '%s'",
-				   fmt_short_channel_id(tmpctx, scid)),
+				   " WHERE short_channel_id = %"PRIu64,
+				   scid.u64),
 			   NULL, NULL, &errmsg);
 	if (err != SQLITE_OK)
 		plugin_err(cmd->plugin, "Could not delete from channels: %s",

tests/test_plugin.py

@@ -4129,7 +4129,7 @@ def test_sql(node_factory, bitcoind):
                   'pubkey': 'BLOB',
                   'secret': 'BLOB',
                   'number': 'REAL',
-                  'short_channel_id': 'TEXT'}
+                  'short_channel_id': 'SCID'}
 
     # Check schemas match
     for table, schema in expected_schemas.items():
@@ -4295,7 +4295,7 @@ def test_sql(node_factory, bitcoind):
     l1.rpc.pay(l3.rpc.invoice(amount_msat=1000000, label='inv1000', description='description 1000 msat')['bolt11'])
 
     # Two channels, l1->l3 *may* have an HTLC in flight.
-    ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', short_channel_id, 'htlcs',"
+    ret = l1.rpc.sql("SELECT json_object('peer_id', hex(pc.peer_id), 'alias', alias, 'scid', fmt_scid(short_channel_id), 'htlcs',"
                      " (SELECT json_group_array(json_object('id', hex(id), 'amount_msat', amount_msat))"
                      " FROM peerchannels_htlcs ph WHERE ph.row = pc.rowid)) FROM peerchannels pc JOIN nodes n"
                      " ON pc.peer_id = n.nodeid ORDER BY n.alias, pc.peer_id;")