wip: add a global result size

Author: avallete

package.json

@@ -36,6 +36,5 @@
     "arrowParens": "always",
     "trailingComma": "es5",
     "singleQuote": true
-  },
-  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
+  }
 }

packages/pg/README.md

@@ -21,45 +21,12 @@ $ npm install pg
 - Pure JavaScript client and native libpq bindings share _the same API_
 - Connection pooling
 - Extensible JS ↔ PostgreSQL data-type coercion
-- Memory safety with configurable result size limits
 - Supported PostgreSQL features
   - Parameterized queries
   - Named statements with query plan caching
   - Async notifications with `LISTEN/NOTIFY`
   - Bulk import & export with `COPY TO/COPY FROM`
 
-### Memory Safety with Result Size Limits
-
-To prevent out-of-memory errors when dealing with unexpectedly large query results, you can set a maximum result size:
-
-```js
-const { Client, Pool } = require('pg')
-
-// For a single client
-const client = new Client({
-  // other connection options
-  maxResultSize: 50 * 1024 * 1024 // 50MB limit
-})
-
-// Or with a pool
-const pool = new Pool({
-  // other connection options
-  maxResultSize: 10 * 1024 * 1024 // 10MB limit
-})
-
-// If a query result exceeds the limit, it will emit an error with code 'RESULT_SIZE_EXCEEDED'
-client.query('SELECT * FROM large_table').catch(err => {
-  if (err.code === 'RESULT_SIZE_EXCEEDED') {
-    console.error(`Query result exceeded size limit of ${err.maxResultSize} bytes`)
-    // Handle gracefully - perhaps use a cursor or add a LIMIT clause
-  } else {
-    // Handle other errors
-  }
-})
-```
-
-For large datasets, consider using [pg-cursor](https://github.com/brianc/node-postgres/tree/master/packages/pg-cursor) to process rows in batches.
-
 ### Extras
 
 node-postgres is by design pretty light on abstractions. These are some handy modules we've been using over the years to complete the picture.

packages/pg/lib/client.js

@@ -52,7 +52,7 @@ class Client extends EventEmitter {
         keepAlive: c.keepAlive || false,
         keepAliveInitialDelayMillis: c.keepAliveInitialDelayMillis || 0,
         encoding: this.connectionParameters.client_encoding || 'utf8',
-        maxResultSize: c.maxResultSize
+        maxResultSize: c.maxResultSize,
       })
     this.queryQueue = []
     this.binary = c.binary || defaults.binary
@@ -244,7 +244,7 @@ class Client extends EventEmitter {
     }
   }
 
-  _handleAuthCleartextPassword(_msg) {
+  _handleAuthCleartextPassword(msg) {
     this._checkPgPass(() => {
       this.connection.password(this.password)
     })
@@ -300,7 +300,7 @@ class Client extends EventEmitter {
     this.secretKey = msg.secretKey
   }
 
-  _handleReadyForQuery(_msg) {
+  _handleReadyForQuery(msg) {
     if (this._connecting) {
       this._connecting = false
       this._connected = true
@@ -377,12 +377,12 @@ class Client extends EventEmitter {
     this.activeQuery.handleDataRow(msg)
   }
 
-  _handlePortalSuspended(_msg) {
+  _handlePortalSuspended(msg) {
     // delegate portalSuspended to active query
     this.activeQuery.handlePortalSuspended(this.connection)
   }
 
-  _handleEmptyQuery(_msg) {
+  _handleEmptyQuery(msg) {
     // delegate emptyQuery to active query
     this.activeQuery.handleEmptyQuery(this.connection)
   }
@@ -411,7 +411,7 @@ class Client extends EventEmitter {
     }
   }
 
-  _handleCopyInResponse(_msg) {
+  _handleCopyInResponse(msg) {
     this.activeQuery.handleCopyInResponse(this.connection)
   }
 

packages/pg/lib/connection.js

@@ -133,55 +133,37 @@ class Connection extends EventEmitter {
   _attachListenersWithSizeLimit(stream) {
     parse(stream, (msg) => {
       var eventName = msg.name === 'error' ? 'errorMessage' : msg.name
-      
+
       // Only track data row messages for result size
       if (msg.name === 'dataRow') {
         // Approximate size by using message length
-        const msgSize = msg.length || this._getApproximateMessageSize(msg)
+        const msgSize = msg.length || 1024 // Default to 1KB if we don't have lenght info
         this._currentResultSize += msgSize
-        
+
         // Check if we've exceeded the max result size
         if (this._currentResultSize > this._maxResultSize) {
           const error = new Error('Query result size exceeded the configured limit')
           error.code = 'RESULT_SIZE_EXCEEDED'
           error.resultSize = this._currentResultSize
           error.maxResultSize = this._maxResultSize
-          this.emit('error', error)
+          this.emit('errorMessage', error)
           this.end() // Terminate the connection
           return
         }
       }
-      
+
       // Reset counter on query completion
       if (msg.name === 'readyForQuery') {
         this._currentResultSize = 0
       }
-      
+
       if (this._emitMessage) {
         this.emit('message', msg)
       }
       this.emit(eventName, msg)
     })
   }
 
-  // Helper method to approximate message size when length is not available
-  _getApproximateMessageSize(msg) {
-    let size = 0
-    if (msg.fields) {
-      // Sum up the sizes of field values
-      msg.fields.forEach(field => {
-        if (field && typeof field === 'string') {
-          size += field.length;
-        } else if (field && typeof field === 'object') {
-          size += JSON.stringify(field).length;
-        } else if (field !== null && field !== undefined) {
-          size += String(field).length;
-        }
-      });
-    }
-    return size > 0 ? size : 1024; // Default to 1KB if we can't determine size
-  }
-
   requestSsl() {
     this.stream.write(serialize.requestSsl())
   }

packages/pg/test/integration/client/max-result-size-tests.js

@@ -4,16 +4,6 @@ var assert = require('assert')
 const { Client } = helper.pg
 const suite = new helper.Suite()
 
-// Just verify the test infrastructure works
-suite.test('sanity check', function (done) {
-  var client = new Client(helper.args)
-  client.connect(assert.success(function () {
-    client.query('SELECT 1 as num', assert.success(function(result) {
-      assert.equal(result.rows.length, 1)
-      client.end(done)
-    }))
-  }))
-})
 
 // Basic test to check if the _maxResultSize property is passed to Connection
 suite.test('client passes maxResultSize to connection', function (done) {
@@ -82,7 +72,7 @@ suite.testAsync('large result triggers error', async () => {
 
   // Start the query but don't await it (it will error)
   const queryPromise = client.query('SELECT repeat(\'x\', 1000) as data FROM generate_series(1, 100)')
-    .catch(err => {
+    .catch(() => {
       // We expect this to error out, silence the rejection
       return null
     })
@@ -94,5 +84,5 @@ suite.testAsync('large result triggers error', async () => {
   await queryPromise
 
   // Clean up
-  await client.end().catch(() => {}) // Ignore errors during cleanup
+  await client.end().catch(() => {})
 }) 

packages/pg/test/integration/connection-pool/result-size-limit-tests.js

@@ -0,0 +1,61 @@
+'use strict'
+
+const helper = require('../test-helper')
+const pg = helper.pg
+var assert = require('assert')
+
+
+const suite = new helper.Suite()
+
+// Test that the pool respects the maxResultSize option
+suite.test('pool respects maxResultSize option', (done) => {
+  const pool = new pg.Pool({
+    ...helper.args,
+    maxResultSize: 1024, // very small limit
+  })
+
+  pool.on('error', (err) => {
+    if (err.code === 'RESULT_SIZE_EXCEEDED') {
+      return done()
+    }
+  })
+
+  const largeQuery = `
+    SELECT generate_series(1, 1000) as num,
+           repeat('x', 100) as data
+  `
+
+  pool.query(largeQuery, (err) => {
+    if (!err) {
+      return done(new Error('Expected query to fail with size limit error'))
+    }
+
+    if (err.code !== 'RESULT_SIZE_EXCEEDED') {
+      return done(new Error(`Expected RESULT_SIZE_EXCEEDED error but got: ${err.message} (${err.code})`))
+    }
+  })
+})
+
+suite.test('pool query works with adequate maxResultSize', (done) => {
+  // Create a pool with a much larger limit
+  const pool = new pg.Pool({
+    ...helper.args,
+    maxResultSize: 100 * 1024,
+  })
+
+  // Use a very simple query that returns a single value
+  const simpleQuery = `SELECT 1 as num`
+
+  // This should succeed
+  pool.query(simpleQuery, (err, result) => {
+    if (err) {
+      return done(err)
+    }
+
+    // Verify we got the expected result
+    assert.deepEqual(result.rows, [{ num: 1 }])
+    // Test passed, clean up
+    pool.end(done)
+  })
+})
+