Skip to content

Commit 3241a69

Browse files
aidtyaclaude
aidtya
and
claude
committed
fix: add noLimit option to driver execute() to prevent silent result truncation
All drivers default to `LIMIT 1001` on SELECT queries and post-truncate to 1000 rows. This silently drops rows when the data-diff engine needs complete result sets — a FULL OUTER JOIN returning >1000 diff rows would be truncated, causing the engine to undercount differences. - Add `ExecuteOptions { noLimit?: boolean }` to the `Connector` interface - When `noLimit: true`, set `effectiveLimit = 0` (falsy) so the existing LIMIT injection guard is skipped, and add `effectiveLimit > 0` to the truncation check so rows aren't sliced to zero - Update all 12 drivers: postgres, clickhouse, snowflake, bigquery, mysql, redshift, databricks, duckdb, oracle, sqlserver, sqlite, mongodb - Pass `{ noLimit: true }` from `data-diff.ts` `executeQuery()` Interactive SQL callers are unaffected — they continue to get the default 1000-row limit. Only the data-diff pipeline opts out. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent b473ce6 commit 3241a69

14 files changed

Lines changed: 66 additions & 55 deletions

File tree

‎packages/drivers/src/bigquery.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* BigQuery driver using the `@google-cloud/bigquery` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let BigQueryModule: any
@@ -37,8 +37,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
3737
client = new BigQuery(options)
3838
},
3939

40-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
41-
const effectiveLimit = limit ?? 1000
40+
async execute(sql: string, limit?: number, binds?: any[], execOptions?: ExecuteOptions): Promise<ConnectorResult> {
41+
const effectiveLimit = execOptions?.noLimit ? 0 : (limit ?? 1000)
4242
const query = sql.replace(/;\s*$/, "")
4343
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
4444

@@ -58,7 +58,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5858

5959
const [rows] = await client.query(options)
6060
const columns = rows.length > 0 ? Object.keys(rows[0]) : []
61-
const truncated = rows.length > effectiveLimit
61+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6262
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
6363

6464
return {

‎packages/drivers/src/clickhouse.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* format for efficient row streaming.
66
*/
77

8-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
8+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
99

1010
export async function connect(config: ConnectionConfig): Promise<Connector> {
1111
const host = (config.host as string) ?? "127.0.0.1"
@@ -68,8 +68,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6868
await httpQuery("SELECT 1")
6969
},
7070

71-
async execute(sql: string, limit?: number): Promise<ConnectorResult> {
72-
const effectiveLimit = limit ?? 1000
71+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
72+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
7373
const isSelectLike = /^\s*(SELECT|WITH)\b/i.test(sql)
7474

7575
let query = sql
@@ -79,7 +79,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
7979

8080
const { columns, rows } = await httpQuery(query)
8181

82-
const truncated = isSelectLike && rows.length > effectiveLimit
82+
const truncated = effectiveLimit > 0 && isSelectLike && rows.length > effectiveLimit
8383
const finalRows = truncated ? rows.slice(0, effectiveLimit) : rows
8484

8585
return {

‎packages/drivers/src/databricks.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Databricks driver using the `@databricks/sql` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let databricksModule: any
@@ -44,8 +44,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4444
})
4545
},
4646

47-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
48-
const effectiveLimit = limit ?? 1000
47+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
48+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4949
let query = sql
5050
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
5151
if (
@@ -65,7 +65,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6565
await operation.close()
6666

6767
const columns = rows.length > 0 ? Object.keys(rows[0]) : []
68-
const truncated = rows.length > effectiveLimit
68+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6969
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
7070

7171
return {

‎packages/drivers/src/duckdb.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* DuckDB driver using the `duckdb` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let duckdb: any
@@ -59,8 +59,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5959
connection = db.connect()
6060
},
6161

62-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
63-
const effectiveLimit = limit ?? 1000
62+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
63+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
6464

6565
let finalSql = sql
6666
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
@@ -77,7 +77,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
7777
: await query(finalSql)
7878
const columns =
7979
rows.length > 0 ? Object.keys(rows[0]) : []
80-
const truncated = rows.length > effectiveLimit
80+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
8181
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
8282

8383
return {

‎packages/drivers/src/mongodb.ts‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* { "database": "mydb", "collection": "users", "command": "countDocuments", "filter": {} }
1515
*/
1616

17-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
17+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
1818

1919
/** Supported MQL commands. */
2020
type MqlCommand =
@@ -241,7 +241,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
241241
await client.connect()
242242
},
243243

244-
async execute(query: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
244+
async execute(query: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
245245
let parsed: MqlQuery
246246
try {
247247
parsed = JSON.parse(query) as MqlQuery
@@ -254,7 +254,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
254254
}
255255

256256
const db = resolveDb(parsed.database)
257-
const effectiveLimit = limit ?? 1000
257+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
258258
const cmd = parsed.command
259259

260260
// Commands that don't need a collection
@@ -304,11 +304,15 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
304304
if (parsed.sort) cursor = cursor.sort(parsed.sort)
305305
if (parsed.skip) cursor = cursor.skip(parsed.skip)
306306
// Cap user-specified limit against effectiveLimit to prevent OOM
307-
const queryLimit = parsed.limit ? Math.min(parsed.limit, effectiveLimit) : effectiveLimit
308-
cursor = cursor.limit(queryLimit + 1)
307+
const queryLimit = effectiveLimit > 0
308+
? (parsed.limit ? Math.min(parsed.limit, effectiveLimit) : effectiveLimit)
309+
: (parsed.limit ?? 0)
310+
if (queryLimit > 0) {
311+
cursor = cursor.limit(queryLimit + 1)
312+
}
309313
const docs = await cursor.toArray()
310314

311-
const truncated = docs.length > queryLimit
315+
const truncated = queryLimit > 0 && docs.length > queryLimit
312316
const limited = truncated ? docs.slice(0, queryLimit) : docs
313317

314318
if (limited.length === 0) {
@@ -336,7 +340,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
336340
// Cap or append $limit to prevent OOM. Skip for $out/$merge write pipelines.
337341
const pipeline = [...parsed.pipeline]
338342
const hasWrite = pipeline.some((stage) => "$out" in stage || "$merge" in stage)
339-
if (!hasWrite) {
343+
if (!hasWrite && effectiveLimit > 0) {
340344
const limitIdx = pipeline.findIndex((stage) => "$limit" in stage)
341345
if (limitIdx >= 0) {
342346
// Cap user-specified $limit against effectiveLimit
@@ -351,7 +355,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
351355

352356
const docs = await coll.aggregate(pipeline).toArray()
353357

354-
const truncated = docs.length > effectiveLimit
358+
const truncated = effectiveLimit > 0 && docs.length > effectiveLimit
355359
const limited = truncated ? docs.slice(0, effectiveLimit) : docs
356360

357361
if (limited.length === 0) {
@@ -386,7 +390,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
386390
throw new Error("distinct requires a 'field' string")
387391
}
388392
const values = await coll.distinct(parsed.field, parsed.filter ?? {})
389-
const truncated = values.length > effectiveLimit
393+
const truncated = effectiveLimit > 0 && values.length > effectiveLimit
390394
const limited = truncated ? values.slice(0, effectiveLimit) : values
391395
return {
392396
columns: [parsed.field],

‎packages/drivers/src/mysql.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* MySQL driver using the `mysql2` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let mysql: any
@@ -41,8 +41,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4141
pool = mysql.createPool(poolConfig)
4242
},
4343

44-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
45-
const effectiveLimit = limit ?? 1000
44+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
45+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4646
let query = sql
4747
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
4848
if (
@@ -56,7 +56,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5656
const [rows, fields] = await pool.query(query)
5757
const columns = fields?.map((f: any) => f.name) ?? []
5858
const rowsArr = Array.isArray(rows) ? rows : []
59-
const truncated = rowsArr.length > effectiveLimit
59+
const truncated = effectiveLimit > 0 && rowsArr.length > effectiveLimit
6060
const limitedRows = truncated
6161
? rowsArr.slice(0, effectiveLimit)
6262
: rowsArr

‎packages/drivers/src/oracle.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Oracle driver using the `oracledb` package (thin mode, pure JS).
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let oracledb: any
@@ -37,8 +37,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
3737
})
3838
},
3939

40-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
41-
const effectiveLimit = limit ?? 1000
40+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
41+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4242
let query = sql
4343
const isSelectLike = /^\s*(SELECT|WITH)\b/i.test(sql)
4444

@@ -61,7 +61,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6161
const columns =
6262
result.metaData?.map((m: any) => m.name) ??
6363
(rows.length > 0 ? Object.keys(rows[0]) : [])
64-
const truncated = rows.length > effectiveLimit
64+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6565
const limitedRows = truncated
6666
? rows.slice(0, effectiveLimit)
6767
: rows

‎packages/drivers/src/postgres.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* PostgreSQL driver using the `pg` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let pg: any
@@ -39,7 +39,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
3939
pool = new Pool(poolConfig)
4040
},
4141

42-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
42+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
4343
const client = await pool.connect()
4444
try {
4545
if (config.statement_timeout) {
@@ -49,7 +49,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4949
}
5050

5151
let query = sql
52-
const effectiveLimit = limit ?? 1000
52+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
5353
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
5454
// Add LIMIT only for SELECT-like queries and if not already present
5555
if (
@@ -62,7 +62,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6262

6363
const result = await client.query(query)
6464
const columns = result.fields?.map((f: any) => f.name) ?? []
65-
const truncated = result.rows.length > effectiveLimit
65+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
6666
const rows = truncated
6767
? result.rows.slice(0, effectiveLimit)
6868
: result.rows

‎packages/drivers/src/redshift.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Uses svv_ system views for introspection.
44
*/
55

6-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
6+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
77

88
export async function connect(config: ConnectionConfig): Promise<Connector> {
99
let pg: any
@@ -40,10 +40,10 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4040
pool = new Pool(poolConfig)
4141
},
4242

43-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
43+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
4444
const client = await pool.connect()
4545
try {
46-
const effectiveLimit = limit ?? 1000
46+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4747
let query = sql
4848
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
4949
if (
@@ -56,7 +56,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5656

5757
const result = await client.query(query)
5858
const columns = result.fields?.map((f: any) => f.name) ?? []
59-
const truncated = result.rows.length > effectiveLimit
59+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
6060
const rows = truncated
6161
? result.rows.slice(0, effectiveLimit)
6262
: result.rows

‎packages/drivers/src/snowflake.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
*/
44

55
import * as fs from "fs"
6-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
6+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
77

88
export async function connect(config: ConnectionConfig): Promise<Connector> {
99
let snowflake: any
@@ -232,8 +232,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
232232
})
233233
},
234234

235-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
236-
const effectiveLimit = limit ?? 1000
235+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
236+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
237237
let query = sql
238238
const isSelectLike = /^\s*(SELECT|WITH|VALUES|SHOW)\b/i.test(sql)
239239
if (
@@ -245,7 +245,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
245245
}
246246

247247
const result = await executeQuery(query, binds)
248-
const truncated = result.rows.length > effectiveLimit
248+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
249249
const rows = truncated
250250
? result.rows.slice(0, effectiveLimit)
251251
: result.rows

0 commit comments

Comments
 (0)