Fix payload size limits and handle entity.too.large in express.json()

.jules/bolt.md

@@ -4,4 +4,11 @@ Learning:
 Swallowing `uncaughtException` and `unhandledRejection` leaves the Node.js process in an undefined and potentially corrupted state, which is a significant reliability and security flaw, especially for a continuously running API gateway.
 
 Action:
-Fixed the global error handlers in `src/index.js` to call `process.exit(1)`, ensuring the process terminates safely and allows a process manager to restart it cleanly. Always ensure future Node.js applications follow this pattern rather than swallowing exceptions.
+Fixed the global error handlers in `src/index.js` to call `process.exit(1)`, ensuring the process terminates safely and allows a process manager to restart it cleanly. Always ensure future Node.js applications follow this pattern rather than swallowing exceptions.
+## 2024-05-18 — Handle Large Payload Sizes for LLM APIS
+
+Learning:
+Unrestricted payload limits can cause Node.js processes to crash or run out of memory when dealing with large LLM prompts. `express.json()` doesn't have a large enough default payload limit for these contexts. Also, when payload size exceeds the limit, Express throws an `entity.too.large` error that needs to be explicitly caught and converted to a 413 response; otherwise, a generic 500 error is returned.
+
+Action:
+Increased the `express.json()` payload limit to `10mb` in `src/index.js` and added a specific error handler for `entity.too.large` errors to return a clean 413 Payload Too Large response.

src/index.js

@@ -20,10 +20,13 @@ process.on('unhandledRejection', (reason, promise) => {
 const app = express();
 app.use(helmet());
 app.use(cors());
-app.use(express.json());
+app.use(express.json({ limit: '10mb' }));
 
 // Handle invalid JSON gracefully
 app.use((err, req, res, next) => {
+  if (err.type === 'entity.too.large') {
+    return res.status(413).json({ error: 'Payload Too Large' });
+  }
   if (err instanceof SyntaxError && err.status === 400 && 'body' in err) {
     return res.status(400).json({ error: 'Invalid JSON payload' });
   }

tests/api.test.js

@@ -46,6 +46,20 @@ test('POST /v1/chat/completions fails with invalid JSON gracefully', async () =>
   assert.strictEqual(res.body.error, 'Invalid JSON payload');
 });
 
+test('POST /v1/chat/completions returns 413 for payload > 10mb', async () => {
+  // Generate a payload larger than 10MB
+  const largeString = 'a'.repeat(10 * 1024 * 1024 + 100);
+  const res = await request(app)
+    .post('/v1/chat/completions')
+    .send({
+      model: 'gpt-4',
+      messages: [{ role: 'user', content: largeString }]
+    });
+
+  assert.strictEqual(res.status, 413);
+  assert.strictEqual(res.body.error, 'Payload Too Large');
+});
+
 test('Generic error handler returns 500 without leaking stack traces', async () => {
   const crypto = require('crypto');
   const originalRandomUUID = crypto.randomUUID;