diff --git a/.jules/bolt.md b/.jules/bolt.md index 1b51e80..bf0ca61 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -4,4 +4,11 @@ Learning: Swallowing `uncaughtException` and `unhandledRejection` leaves the Node.js process in an undefined and potentially corrupted state, which is a significant reliability and security flaw, especially for a continuously running API gateway. Action: -Fixed the global error handlers in `src/index.js` to call `process.exit(1)`, ensuring the process terminates safely and allows a process manager to restart it cleanly. Always ensure future Node.js applications follow this pattern rather than swallowing exceptions. \ No newline at end of file +Fixed the global error handlers in `src/index.js` to call `process.exit(1)`, ensuring the process terminates safely and allows a process manager to restart it cleanly. Always ensure future Node.js applications follow this pattern rather than swallowing exceptions. +## 2024-05-18 — Handle Large Payload Sizes for LLM APIS + +Learning: +Unrestricted payload limits can cause Node.js processes to crash or run out of memory when dealing with large LLM prompts. `express.json()` doesn't have a large enough default payload limit for these contexts. Also, when payload size exceeds the limit, Express throws an `entity.too.large` error that needs to be explicitly caught and converted to a 413 response; otherwise, a generic 500 error is returned. + +Action: +Increased the `express.json()` payload limit to `10mb` in `src/index.js` and added a specific error handler for `entity.too.large` errors to return a clean 413 Payload Too Large response. diff --git a/src/index.js b/src/index.js index 5332530..e01d9e5 100644 --- a/src/index.js +++ b/src/index.js @@ -20,10 +20,13 @@ process.on('unhandledRejection', (reason, promise) => { const app = express(); app.use(helmet()); app.use(cors()); -app.use(express.json()); +app.use(express.json({ limit: '10mb' })); // Handle invalid JSON gracefully app.use((err, req, res, next) => { + if (err.type === 'entity.too.large') { + return res.status(413).json({ error: 'Payload Too Large' }); + } if (err instanceof SyntaxError && err.status === 400 && 'body' in err) { return res.status(400).json({ error: 'Invalid JSON payload' }); } diff --git a/tests/api.test.js b/tests/api.test.js index e184779..0d310b9 100644 --- a/tests/api.test.js +++ b/tests/api.test.js @@ -46,6 +46,20 @@ test('POST /v1/chat/completions fails with invalid JSON gracefully', async () => assert.strictEqual(res.body.error, 'Invalid JSON payload'); }); +test('POST /v1/chat/completions returns 413 for payload > 10mb', async () => { + // Generate a payload larger than 10MB + const largeString = 'a'.repeat(10 * 1024 * 1024 + 100); + const res = await request(app) + .post('/v1/chat/completions') + .send({ + model: 'gpt-4', + messages: [{ role: 'user', content: largeString }] + }); + + assert.strictEqual(res.status, 413); + assert.strictEqual(res.body.error, 'Payload Too Large'); +}); + test('Generic error handler returns 500 without leaking stack traces', async () => { const crypto = require('crypto'); const originalRandomUUID = crypto.randomUUID;