diff --git a/.changeset/include-column-names-d1-export.md b/.changeset/include-column-names-d1-export.md new file mode 100644 index 0000000000..8afd2008c1 --- /dev/null +++ b/.changeset/include-column-names-d1-export.md @@ -0,0 +1,7 @@ +--- +"wrangler": patch +--- + +Include column names in D1 SQL export INSERT statements + +D1 SQL exports now include column names in INSERT statements (e.g., `INSERT INTO "table" ("col1","col2") VALUES(...)`). This ensures that exported SQL can be successfully imported even when the target table has columns in a different order than the original, which commonly occurs during iterative development when schemas evolve. diff --git a/packages/miniflare/src/workers/d1/dumpSql.ts b/packages/miniflare/src/workers/d1/dumpSql.ts index 2a027f8677..8624e5a979 100644 --- a/packages/miniflare/src/workers/d1/dumpSql.ts +++ b/packages/miniflare/src/workers/d1/dumpSql.ts @@ -5,6 +5,23 @@ // as possible, with any deviations noted. import type { SqlStorage } from "@cloudflare/workers-types/experimental"; +/** Stats tracking for dumpSql. Will be mutated in place if provided. */ +export interface DumpSqlStats { + rows_read: number; + rows_written: number; + // Stats for tracking INSERT statement sizes (column names are always included) + /** Number of INSERT statements over 100KB (current size, with column names) */ + inserts_over_100kb_with_column_names: number; + /** Number of INSERT statements that would be over 100KB without column names (for backward comparison) */ + inserts_already_over_100kb: number; + /** Total number of INSERT statements generated */ + total_inserts: number; + /** Maximum INSERT statement size without column names (hypothetical, for backward comparison) */ + max_insert_size: number; + /** Maximum INSERT statement size (current size, with column names) */ + max_insert_size_with_column_names: number; +} + export function* dumpSql( db: SqlStorage, options?: { @@ -13,7 +30,7 @@ export function* dumpSql( tables?: string[]; }, /** Optional stats tracking. Will be mutated in place if provided */ - stats?: { rows_read: number; rows_written: number } + stats?: DumpSqlStats ) { // WARNING: the caller in D1 assumes non-empty exports, so think carefully before removing this initial yield. yield `PRAGMA defer_foreign_keys=TRUE;`; @@ -83,6 +100,9 @@ export function* dumpSql( const select = `SELECT ${columns.map((c) => escapeId(c.name)).join(", ")} FROM ${escapeId(table)};`; const rows_cursor = db.exec(select); + const columnNames = columns.map((c) => escapeId(c.name)).join(","); + // The column names portion is: " (" + columnNames + ")" = 3 + columnNames.length + const columnNamesOverhead = 3 + columnNames.length; for (const dataRow of rows_cursor.raw()) { const formattedCells = dataRow.map((cell: unknown, i: number) => { const colType = columns[i].type; @@ -109,7 +129,31 @@ export function* dumpSql( } }); - yield `INSERT INTO ${escapeId(table)} VALUES(${formattedCells.join(",")});`; + const insertStmt = `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`; + + // Track stats for INSERT statement sizes + if (stats) { + const currentSize = insertStmt.length; + // Calculate what the size would be without column names (for comparison) + const sizeWithoutColumnNames = currentSize - columnNamesOverhead; + const LIMIT = 100 * 1024; // 100KB + + stats.total_inserts++; + if (sizeWithoutColumnNames > LIMIT) { + stats.inserts_already_over_100kb++; + } + if (currentSize > LIMIT) { + stats.inserts_over_100kb_with_column_names++; + } + if (sizeWithoutColumnNames > stats.max_insert_size) { + stats.max_insert_size = sizeWithoutColumnNames; + } + if (currentSize > stats.max_insert_size_with_column_names) { + stats.max_insert_size_with_column_names = currentSize; + } + } + + yield insertStmt; } if (stats) { stats.rows_read += rows_cursor.rowsRead; diff --git a/packages/wrangler/src/__tests__/d1/export.test.ts b/packages/wrangler/src/__tests__/d1/export.test.ts index c52818e18d..6c73b3df99 100644 --- a/packages/wrangler/src/__tests__/d1/export.test.ts +++ b/packages/wrangler/src/__tests__/d1/export.test.ts @@ -73,14 +73,14 @@ describe("export", () => { const create_foo = "CREATE TABLE foo(id INTEGER PRIMARY KEY, value TEXT);"; const create_bar = "CREATE TABLE bar(id INTEGER PRIMARY KEY, value TEXT);"; const insert_foo = [ - `INSERT INTO "foo" VALUES(1,'xxx');`, - `INSERT INTO "foo" VALUES(2,'yyy');`, - `INSERT INTO "foo" VALUES(3,'zzz');`, + `INSERT INTO "foo" ("id","value") VALUES(1,'xxx');`, + `INSERT INTO "foo" ("id","value") VALUES(2,'yyy');`, + `INSERT INTO "foo" ("id","value") VALUES(3,'zzz');`, ]; const insert_bar = [ - `INSERT INTO "bar" VALUES(1,'aaa');`, - `INSERT INTO "bar" VALUES(2,'bbb');`, - `INSERT INTO "bar" VALUES(3,'ccc');`, + `INSERT INTO "bar" ("id","value") VALUES(1,'aaa');`, + `INSERT INTO "bar" ("id","value") VALUES(2,'bbb');`, + `INSERT INTO "bar" ("id","value") VALUES(3,'ccc');`, ]; // Full export @@ -311,13 +311,13 @@ describe("export", () => { [ "PRAGMA defer_foreign_keys=TRUE;", "CREATE TABLE foo(id INTEGER PRIMARY KEY, value TEXT);", - "INSERT INTO \"foo\" VALUES(1,'xxx');", - "INSERT INTO \"foo\" VALUES(2,'yyy');", - "INSERT INTO \"foo\" VALUES(3,'zzz');", + 'INSERT INTO "foo" ("id","value") VALUES(1,\'xxx\');', + 'INSERT INTO "foo" ("id","value") VALUES(2,\'yyy\');', + 'INSERT INTO "foo" ("id","value") VALUES(3,\'zzz\');', "CREATE TABLE baz(id INTEGER PRIMARY KEY, value TEXT);", - "INSERT INTO \"baz\" VALUES(1,'111');", - "INSERT INTO \"baz\" VALUES(2,'222');", - "INSERT INTO \"baz\" VALUES(3,'333');", + 'INSERT INTO "baz" ("id","value") VALUES(1,\'111\');', + 'INSERT INTO "baz" ("id","value") VALUES(2,\'222\');', + 'INSERT INTO "baz" ("id","value") VALUES(3,\'333\');', ].join("\n") ); });