[wrangler] Include column names in D1 SQL export (#12277)

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: Somhairle MacLeòid <penalosa@users.noreply.github.com>
Co-authored-by: David Laban <dlaban@cloudflare.com>
Co-authored-by: Pete Bacon Darwin <pbacondarwin@cloudflare.com>

Author: 4 people

.changeset/include-column-names-d1-export.md

@@ -0,0 +1,7 @@
+---
+"wrangler": patch
+---
+
+Include column names in D1 SQL export INSERT statements
+
+D1 SQL exports now include column names in INSERT statements (e.g., `INSERT INTO "table" ("col1","col2") VALUES(...)`). This ensures that exported SQL can be successfully imported even when the target table has columns in a different order than the original, which commonly occurs during iterative development when schemas evolve.

packages/miniflare/src/workers/d1/dumpSql.ts

@@ -5,6 +5,23 @@
 // as possible, with any deviations noted.
 import type { SqlStorage } from "@cloudflare/workers-types/experimental";
 
+/** Stats tracking for dumpSql. Will be mutated in place if provided. */
+export interface DumpSqlStats {
+	rows_read: number;
+	rows_written: number;
+	// Stats for tracking INSERT statement sizes (column names are always included)
+	/** Number of INSERT statements over 100KB (current size, with column names) */
+	inserts_over_100kb_with_column_names: number;
+	/** Number of INSERT statements that would be over 100KB without column names (for backward comparison) */
+	inserts_already_over_100kb: number;
+	/** Total number of INSERT statements generated */
+	total_inserts: number;
+	/** Maximum INSERT statement size without column names (hypothetical, for backward comparison) */
+	max_insert_size: number;
+	/** Maximum INSERT statement size (current size, with column names) */
+	max_insert_size_with_column_names: number;
+}
+
 export function* dumpSql(
 	db: SqlStorage,
 	options?: {
@@ -13,7 +30,7 @@ export function* dumpSql(
 		tables?: string[];
 	},
 	/** Optional stats tracking. Will be mutated in place if provided */
-	stats?: { rows_read: number; rows_written: number }
+	stats?: DumpSqlStats
 ) {
 	// WARNING: the caller in D1 assumes non-empty exports, so think carefully before removing this initial yield.
 	yield `PRAGMA defer_foreign_keys=TRUE;`;
@@ -83,6 +100,9 @@ export function* dumpSql(
 
 		const select = `SELECT ${columns.map((c) => escapeId(c.name)).join(", ")} FROM ${escapeId(table)};`;
 		const rows_cursor = db.exec(select);
+		const columnNames = columns.map((c) => escapeId(c.name)).join(",");
+		// The column names portion is: " (" + columnNames + ")" = 3 + columnNames.length
+		const columnNamesOverhead = 3 + columnNames.length;
 		for (const dataRow of rows_cursor.raw()) {
 			const formattedCells = dataRow.map((cell: unknown, i: number) => {
 				const colType = columns[i].type;
@@ -109,7 +129,31 @@ export function* dumpSql(
 				}
 			});
 
-			yield `INSERT INTO ${escapeId(table)} VALUES(${formattedCells.join(",")});`;
+			const insertStmt = `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`;
+
+			// Track stats for INSERT statement sizes
+			if (stats) {
+				const currentSize = insertStmt.length;
+				// Calculate what the size would be without column names (for comparison)
+				const sizeWithoutColumnNames = currentSize - columnNamesOverhead;
+				const LIMIT = 100 * 1024; // 100KB
+
+				stats.total_inserts++;
+				if (sizeWithoutColumnNames > LIMIT) {
+					stats.inserts_already_over_100kb++;
+				}
+				if (currentSize > LIMIT) {
+					stats.inserts_over_100kb_with_column_names++;
+				}
+				if (sizeWithoutColumnNames > stats.max_insert_size) {
+					stats.max_insert_size = sizeWithoutColumnNames;
+				}
+				if (currentSize > stats.max_insert_size_with_column_names) {
+					stats.max_insert_size_with_column_names = currentSize;
+				}
+			}
+
+			yield insertStmt;
 		}
 		if (stats) {
 			stats.rows_read += rows_cursor.rowsRead;

packages/wrangler/src/__tests__/d1/export.test.ts

@@ -73,14 +73,14 @@ describe("export", () => {
 		const create_foo = "CREATE TABLE foo(id INTEGER PRIMARY KEY, value TEXT);";
 		const create_bar = "CREATE TABLE bar(id INTEGER PRIMARY KEY, value TEXT);";
 		const insert_foo = [
-			`INSERT INTO "foo" VALUES(1,'xxx');`,
-			`INSERT INTO "foo" VALUES(2,'yyy');`,
-			`INSERT INTO "foo" VALUES(3,'zzz');`,
+			`INSERT INTO "foo" ("id","value") VALUES(1,'xxx');`,
+			`INSERT INTO "foo" ("id","value") VALUES(2,'yyy');`,
+			`INSERT INTO "foo" ("id","value") VALUES(3,'zzz');`,
 		];
 		const insert_bar = [
-			`INSERT INTO "bar" VALUES(1,'aaa');`,
-			`INSERT INTO "bar" VALUES(2,'bbb');`,
-			`INSERT INTO "bar" VALUES(3,'ccc');`,
+			`INSERT INTO "bar" ("id","value") VALUES(1,'aaa');`,
+			`INSERT INTO "bar" ("id","value") VALUES(2,'bbb');`,
+			`INSERT INTO "bar" ("id","value") VALUES(3,'ccc');`,
 		];
 
 		// Full export
@@ -311,13 +311,13 @@ describe("export", () => {
 			[
 				"PRAGMA defer_foreign_keys=TRUE;",
 				"CREATE TABLE foo(id INTEGER PRIMARY KEY, value TEXT);",
-				"INSERT INTO \"foo\" VALUES(1,'xxx');",
-				"INSERT INTO \"foo\" VALUES(2,'yyy');",
-				"INSERT INTO \"foo\" VALUES(3,'zzz');",
+				'INSERT INTO "foo" ("id","value") VALUES(1,\'xxx\');',
+				'INSERT INTO "foo" ("id","value") VALUES(2,\'yyy\');',
+				'INSERT INTO "foo" ("id","value") VALUES(3,\'zzz\');',
 				"CREATE TABLE baz(id INTEGER PRIMARY KEY, value TEXT);",
-				"INSERT INTO \"baz\" VALUES(1,'111');",
-				"INSERT INTO \"baz\" VALUES(2,'222');",
-				"INSERT INTO \"baz\" VALUES(3,'333');",
+				'INSERT INTO "baz" ("id","value") VALUES(1,\'111\');',
+				'INSERT INTO "baz" ("id","value") VALUES(2,\'222\');',
+				'INSERT INTO "baz" ("id","value") VALUES(3,\'333\');',
 			].join("\n")
 		);
 	});