diff --git a/.changeset/humble-ravens-send.md b/.changeset/humble-ravens-send.md new file mode 100644 index 000000000..94456009d --- /dev/null +++ b/.changeset/humble-ravens-send.md @@ -0,0 +1,5 @@ +--- +'@openfn/logger': patch +--- + +Fix an issue where logger.proxy can throw for nil values diff --git a/.changeset/odd-sloths-make.md b/.changeset/odd-sloths-make.md new file mode 100644 index 000000000..5fcdb347c --- /dev/null +++ b/.changeset/odd-sloths-make.md @@ -0,0 +1,6 @@ +--- +'@openfn/engine-multi': patch +'@openfn/ws-worker': patch +--- + +Use streaming algorithm to calculate payload size diff --git a/packages/engine-multi/src/util/ensure-payload-size.ts b/packages/engine-multi/src/util/ensure-payload-size.ts index db6dc6ee3..4401b0707 100644 --- a/packages/engine-multi/src/util/ensure-payload-size.ts +++ b/packages/engine-multi/src/util/ensure-payload-size.ts @@ -48,6 +48,16 @@ export const calculateSizeStream = async ( value: any, limit?: number ): Promise => { + // skip all primitives + if ( + !value || + typeof value === 'number' || + typeof value === 'boolean' || + typeof value === 'function' + ) { + return 1; + } + let size_bytes = 0; const stream = new JsonStreamStringify(value); @@ -69,11 +79,16 @@ export default async (payload: any, limit_mb: number = 10) => { const newPayload = { ...payload }; for (const key of KEYS_TO_VERIFY) { - try { - await verify(payload[key], limit_mb); - } catch (e) { - Object.assign(newPayload[key], replacements[key] ?? replacements.default); - newPayload.redacted = true; + if (key in payload) { + try { + await verify(payload[key], limit_mb, 'stream'); + } catch (e) { + Object.assign( + newPayload[key], + replacements[key] ?? replacements.default + ); + newPayload.redacted = true; + } } } diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index 8ec2a9ea6..88cf255f9 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -180,7 +180,7 @@ test.serial('should emit a job-complete event', async (t) => { t.assert(!isNaN(event.duration)); }); -test.serial('should emit a log event', async (t) => { +test.serial.only('should emit a log event', async (t) => { let workflowLog: any; const plan = { id: 'y', diff --git a/packages/logger/src/logger.ts b/packages/logger/src/logger.ts index ab064efa1..245a94fca 100644 --- a/packages/logger/src/logger.ts +++ b/packages/logger/src/logger.ts @@ -262,6 +262,8 @@ export default function (name?: string, options: LogOptions = {}): Logger { } j = j as JSONLog; + j.message ??= []; + log(j.name, j.level, ...j.message); return [j.name, j.level, ...j.message]; };