Skip to content

Commit 609e2fc

Browse files
committed
fix(miniflare): add DumpSqlStats interface and INSERT size tracking
Add stats tracking for INSERT statement sizes to help analyze the impact of including column names in D1 SQL exports. This syncs the miniflare dumpSql.ts with the d1-worker version from GitLab MR 1056.
1 parent 2038ca6 commit 609e2fc

1 file changed

Lines changed: 45 additions & 2 deletions

File tree

packages/miniflare/src/workers/d1/dumpSql.ts

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,23 @@
55
// as possible, with any deviations noted.
66
import { SqlStorage } from "@cloudflare/workers-types/experimental";
77

8+
/** Stats tracking for dumpSql. Will be mutated in place if provided. */
9+
export interface DumpSqlStats {
10+
rows_read: number;
11+
rows_written: number;
12+
// Stats for tracking INSERT statement sizes (column names are always included)
13+
/** Number of INSERT statements over 100KB (current size, with column names) */
14+
inserts_over_100kb_with_column_names: number;
15+
/** Number of INSERT statements that would be over 100KB without column names (for backward comparison) */
16+
inserts_already_over_100kb: number;
17+
/** Total number of INSERT statements generated */
18+
total_inserts: number;
19+
/** Maximum INSERT statement size without column names (hypothetical, for backward comparison) */
20+
max_insert_size: number;
21+
/** Maximum INSERT statement size (current size, with column names) */
22+
max_insert_size_with_column_names: number;
23+
}
24+
825
export function* dumpSql(
926
db: SqlStorage,
1027
options?: {
@@ -13,7 +30,7 @@ export function* dumpSql(
1330
tables?: string[];
1431
},
1532
/** Optional stats tracking. Will be mutated in place if provided */
16-
stats?: { rows_read: number; rows_written: number }
33+
stats?: DumpSqlStats
1734
) {
1835
// WARNING: the caller in D1 assumes non-empty exports, so think carefully before removing this initial yield.
1936
yield `PRAGMA defer_foreign_keys=TRUE;`;
@@ -84,6 +101,8 @@ export function* dumpSql(
84101
const select = `SELECT ${columns.map((c) => escapeId(c.name)).join(", ")} FROM ${escapeId(table)};`;
85102
const rows_cursor = db.exec(select);
86103
const columnNames = columns.map((c) => escapeId(c.name)).join(",");
104+
// The column names portion is: " (" + columnNames + ")" = 3 + columnNames.length
105+
const columnNamesOverhead = 3 + columnNames.length;
87106
for (const dataRow of rows_cursor.raw()) {
88107
const formattedCells = dataRow.map((cell: unknown, i: number) => {
89108
const colType = columns[i].type;
@@ -110,7 +129,31 @@ export function* dumpSql(
110129
}
111130
});
112131

113-
yield `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`;
132+
const insertStmt = `INSERT INTO ${escapeId(table)} (${columnNames}) VALUES(${formattedCells.join(",")});`;
133+
134+
// Track stats for INSERT statement sizes
135+
if (stats) {
136+
const currentSize = insertStmt.length;
137+
// Calculate what the size would be without column names (for comparison)
138+
const sizeWithoutColumnNames = currentSize - columnNamesOverhead;
139+
const LIMIT = 100 * 1024; // 100KB
140+
141+
stats.total_inserts++;
142+
if (sizeWithoutColumnNames > LIMIT) {
143+
stats.inserts_already_over_100kb++;
144+
}
145+
if (currentSize > LIMIT) {
146+
stats.inserts_over_100kb_with_column_names++;
147+
}
148+
if (sizeWithoutColumnNames > stats.max_insert_size) {
149+
stats.max_insert_size = sizeWithoutColumnNames;
150+
}
151+
if (currentSize > stats.max_insert_size_with_column_names) {
152+
stats.max_insert_size_with_column_names = currentSize;
153+
}
154+
}
155+
156+
yield insertStmt;
114157
}
115158
if (stats) {
116159
stats.rows_read += rows_cursor.rowsRead;

0 commit comments

Comments
 (0)