Skip to content

Commit 4131ea9

Browse files
aidtyaclaude
authored and
suryaiyer95
committed
fix: add noLimit option to driver execute() to prevent silent result truncation
All drivers default to `LIMIT 1001` on SELECT queries and post-truncate to 1000 rows. This silently drops rows when the data-diff engine needs complete result sets — a FULL OUTER JOIN returning >1000 diff rows would be truncated, causing the engine to undercount differences. - Add `ExecuteOptions { noLimit?: boolean }` to the `Connector` interface - When `noLimit: true`, set `effectiveLimit = 0` (falsy) so the existing LIMIT injection guard is skipped, and add `effectiveLimit > 0` to the truncation check so rows aren't sliced to zero - Update all 12 drivers: postgres, clickhouse, snowflake, bigquery, mysql, redshift, databricks, duckdb, oracle, sqlserver, sqlite, mongodb - Pass `{ noLimit: true }` from `data-diff.ts` `executeQuery()` Interactive SQL callers are unaffected — they continue to get the default 1000-row limit. Only the data-diff pipeline opts out. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 9ed95bb commit 4131ea9

File tree

12 files changed

+49
-42
lines changed

12 files changed

+49
-42
lines changed

‎packages/drivers/src/bigquery.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* BigQuery driver using the `@google-cloud/bigquery` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let BigQueryModule: any
@@ -37,8 +37,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
3737
client = new BigQuery(options)
3838
},
3939

40-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
41-
const effectiveLimit = limit ?? 1000
40+
async execute(sql: string, limit?: number, binds?: any[], execOptions?: ExecuteOptions): Promise<ConnectorResult> {
41+
const effectiveLimit = execOptions?.noLimit ? 0 : (limit ?? 1000)
4242
const query = sql.replace(/;\s*$/, "")
4343
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
4444

@@ -58,7 +58,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5858

5959
const [rows] = await client.query(options)
6060
const columns = rows.length > 0 ? Object.keys(rows[0]) : []
61-
const truncated = rows.length > effectiveLimit
61+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6262
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
6363

6464
return {

‎packages/drivers/src/databricks.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Databricks driver using the `@databricks/sql` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let databricksModule: any
@@ -44,8 +44,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4444
})
4545
},
4646

47-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
48-
const effectiveLimit = limit ?? 1000
47+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
48+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4949
let query = sql
5050
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
5151
if (
@@ -65,7 +65,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6565
await operation.close()
6666

6767
const columns = rows.length > 0 ? Object.keys(rows[0]) : []
68-
const truncated = rows.length > effectiveLimit
68+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6969
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
7070

7171
return {

‎packages/drivers/src/duckdb.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* DuckDB driver using the `duckdb` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let duckdb: any
@@ -105,8 +105,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
105105
connection = db.connect()
106106
},
107107

108-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
109-
const effectiveLimit = limit ?? 1000
108+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
109+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
110110

111111
let finalSql = sql
112112
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
@@ -123,7 +123,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
123123
: await query(finalSql)
124124
const columns =
125125
rows.length > 0 ? Object.keys(rows[0]) : []
126-
const truncated = rows.length > effectiveLimit
126+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
127127
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
128128

129129
return {

‎packages/drivers/src/mysql.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* MySQL driver using the `mysql2` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let mysql: any
@@ -41,8 +41,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4141
pool = mysql.createPool(poolConfig)
4242
},
4343

44-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
45-
const effectiveLimit = limit ?? 1000
44+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
45+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4646
let query = sql
4747
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
4848
if (
@@ -56,7 +56,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5656
const [rows, fields] = await pool.query(query)
5757
const columns = fields?.map((f: any) => f.name) ?? []
5858
const rowsArr = Array.isArray(rows) ? rows : []
59-
const truncated = rowsArr.length > effectiveLimit
59+
const truncated = effectiveLimit > 0 && rowsArr.length > effectiveLimit
6060
const limitedRows = truncated
6161
? rowsArr.slice(0, effectiveLimit)
6262
: rowsArr

‎packages/drivers/src/oracle.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* Oracle driver using the `oracledb` package (thin mode, pure JS).
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let oracledb: any
@@ -37,8 +37,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
3737
})
3838
},
3939

40-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
41-
const effectiveLimit = limit ?? 1000
40+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
41+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4242
let query = sql
4343
const isSelectLike = /^\s*(SELECT|WITH)\b/i.test(sql)
4444

@@ -61,7 +61,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6161
const columns =
6262
result.metaData?.map((m: any) => m.name) ??
6363
(rows.length > 0 ? Object.keys(rows[0]) : [])
64-
const truncated = rows.length > effectiveLimit
64+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6565
const limitedRows = truncated
6666
? rows.slice(0, effectiveLimit)
6767
: rows

‎packages/drivers/src/postgres.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* PostgreSQL driver using the `pg` package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let pg: any
@@ -46,7 +46,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4646
pool = new Pool(poolConfig)
4747
},
4848

49-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
49+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
5050
const client = await pool.connect()
5151
try {
5252
if (config.statement_timeout) {
@@ -57,7 +57,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
5757
}
5858

5959
let query = sql
60-
const effectiveLimit = limit ?? 1000
60+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
6161
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
6262
// Add LIMIT only for SELECT-like queries and if not already present
6363
if (
@@ -70,7 +70,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
7070

7171
const result = await client.query(query)
7272
const columns = result.fields?.map((f: any) => f.name) ?? []
73-
const truncated = result.rows.length > effectiveLimit
73+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
7474
const rows = truncated
7575
? result.rows.slice(0, effectiveLimit)
7676
: result.rows

‎packages/drivers/src/redshift.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Uses svv_ system views for introspection.
44
*/
55

6-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
6+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
77

88
export async function connect(config: ConnectionConfig): Promise<Connector> {
99
let pg: any
@@ -46,10 +46,10 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4646
pool = new Pool(poolConfig)
4747
},
4848

49-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
49+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
5050
const client = await pool.connect()
5151
try {
52-
const effectiveLimit = limit ?? 1000
52+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
5353
let query = sql
5454
const isSelectLike = /^\s*(SELECT|WITH|VALUES)\b/i.test(sql)
5555
if (
@@ -62,7 +62,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6262

6363
const result = await client.query(query)
6464
const columns = result.fields?.map((f: any) => f.name) ?? []
65-
const truncated = result.rows.length > effectiveLimit
65+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
6666
const rows = truncated
6767
? result.rows.slice(0, effectiveLimit)
6868
: result.rows

‎packages/drivers/src/snowflake.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
*/
44

55
import * as fs from "fs"
6-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
6+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
77

88
export async function connect(config: ConnectionConfig): Promise<Connector> {
99
let snowflake: any
@@ -232,8 +232,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
232232
})
233233
},
234234

235-
async execute(sql: string, limit?: number, binds?: any[]): Promise<ConnectorResult> {
236-
const effectiveLimit = limit ?? 1000
235+
async execute(sql: string, limit?: number, binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
236+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
237237
let query = sql
238238
const isSelectLike = /^\s*(SELECT|WITH|VALUES|SHOW)\b/i.test(sql)
239239
if (
@@ -245,7 +245,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
245245
}
246246

247247
const result = await executeQuery(query, binds)
248-
const truncated = result.rows.length > effectiveLimit
248+
const truncated = effectiveLimit > 0 && result.rows.length > effectiveLimit
249249
const rows = truncated
250250
? result.rows.slice(0, effectiveLimit)
251251
: result.rows

‎packages/drivers/src/sqlite.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
*/
55

66
import { Database } from "bun:sqlite"
7-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
7+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
88

99
export async function connect(config: ConnectionConfig): Promise<Connector> {
1010
const dbPath = (config.path as string) ?? ":memory:"
@@ -22,9 +22,9 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
2222
}
2323
},
2424

25-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
25+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
2626
if (!db) throw new Error("SQLite connection not open")
27-
const effectiveLimit = limit ?? 1000
27+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
2828

2929
// Determine if this is a SELECT-like statement
3030
const trimmed = sql.trim().toLowerCase()
@@ -60,7 +60,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6060
const stmt = db.prepare(query)
6161
const rows = stmt.all() as any[]
6262
const columns = rows.length > 0 ? Object.keys(rows[0]) : []
63-
const truncated = rows.length > effectiveLimit
63+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
6464
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
6565

6666
return {

‎packages/drivers/src/sqlserver.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* SQL Server driver using the `mssql` (tedious) package.
33
*/
44

5-
import type { ConnectionConfig, Connector, ConnectorResult, SchemaColumn } from "./types"
5+
import type { ConnectionConfig, Connector, ConnectorResult, ExecuteOptions, SchemaColumn } from "./types"
66

77
export async function connect(config: ConnectionConfig): Promise<Connector> {
88
let mssql: any
@@ -42,8 +42,8 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
4242
pool = await mssql.connect(mssqlConfig)
4343
},
4444

45-
async execute(sql: string, limit?: number, _binds?: any[]): Promise<ConnectorResult> {
46-
const effectiveLimit = limit ?? 1000
45+
async execute(sql: string, limit?: number, _binds?: any[], options?: ExecuteOptions): Promise<ConnectorResult> {
46+
const effectiveLimit = options?.noLimit ? 0 : (limit ?? 1000)
4747

4848
let query = sql
4949
const isSelectLike = /^\s*SELECT\b/i.test(sql)
@@ -69,7 +69,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
6969
: (result.recordset?.columns
7070
? Object.keys(result.recordset.columns)
7171
: [])
72-
const truncated = rows.length > effectiveLimit
72+
const truncated = effectiveLimit > 0 && rows.length > effectiveLimit
7373
const limitedRows = truncated ? rows.slice(0, effectiveLimit) : rows
7474

7575
return {

0 commit comments

Comments
 (0)