Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions workers/grouper/src/data-filter.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
import type { EventAddons, EventData } from '@hawk.so/types';
import { unsafeFields } from '../../../lib/utils/unsafeFields';
import { rightTrim } from '../../../lib/utils/string';

/**
* Maximum depth for object traversal to prevent excessive memory allocations
*/
const MAX_TRAVERSAL_DEPTH = 20;

/**
* Maximum length for event title before appending ellipsis
*/
const MAX_TITLE_LENGTH = 400;

/**
* Recursively iterate through object and call function on each key
*
Expand Down Expand Up @@ -135,13 +141,27 @@ export default class DataFilter {
* @param event - event to process
*/
public processEvent(event: EventData<EventAddons>): void {
this.trimEventTitle(event);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now we sanitize only title, but long strings (as well as very deep objects, long arrays) can be included in other fields:

  • title
  • context
  • addends
  • backtrace[].arguments
  • breadcrumbs[].message
  • breadcrumbs[].data
  • breadcrumbs[].message

I'd suggest to add the Sanitizer utility like we have in Hawk JavaScript. And use it here in DataFilter.


unsafeFields.forEach(field => {
if (event[field]) {
this.processField(event[field]);
}
});
}

/**
* Trim event title to the maximum allowed length.
* It mutates the original object.
*
* @param event - event to process
*/
public trimEventTitle(event: EventData<EventAddons>): void {
if (typeof event.title === 'string') {
event.title = rightTrim(event.title, MAX_TITLE_LENGTH);
}
}

/**
* Recursively iterates object and applies filtering to its entries
*
Expand Down
16 changes: 8 additions & 8 deletions workers/grouper/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ const DB_DUPLICATE_KEY_ERROR = '11000';
const DAILY_METRICS_RETENTION_DAYS = 90;

/**
* Maximum length for backtrace code line or title
* Maximum length for backtrace code line
*/
const MAX_CODE_LINE_LENGTH = 140;

Expand Down Expand Up @@ -198,8 +198,6 @@ export default class GrouperWorker extends Worker {
this.grouperMetrics.observePayloadSize(taskPayloadSize);
this.memoryMonitor.logBeforeHandle(memoryBeforeHandle, handledTasksCount, taskPayloadSize, task.projectId);

this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`);

// FIX RELEASE TYPE
// TODO: REMOVE AFTER 01.01.2026, after the most of the users update to new js catcher
if (task.payload && task.payload.release !== undefined) {
Expand All @@ -209,6 +207,13 @@ export default class GrouperWorker extends Worker {
};
}

/**
* Filter event data before logging and hashing so logs, hash and stored event stay consistent.
*/
this.dataFilter.processEvent(task.payload);

this.logger.info(`[handle] project=${task.projectId} catcher=${task.catcherType} title="${task.payload.title}" payloadSize=${taskPayloadSize}b backtraceFrames=${task.payload.backtrace?.length ?? 0}`);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this log is needed? It will be printed for every event


let uniqueEventHash = await session.measureStep('hash', () => this.getUniqueEventHash(task));
let existedEvent: GroupedEventDBScheme;
let repetitionId = null;
Expand All @@ -219,11 +224,6 @@ export default class GrouperWorker extends Worker {
* Trim source code lines to prevent memory leaks
*/
this.trimSourceCodeLines(task.payload);

/**
* Filter sensitive information
*/
this.dataFilter.processEvent(task.payload);
});

/**
Expand Down
14 changes: 14 additions & 0 deletions workers/grouper/tests/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,20 @@ describe('GrouperWorker', () => {
expect(await eventsCollection.find().count()).toBe(1);
});

test('Should trim event title to the maximum length before saving', async () => {
const longTitle = 'A'.repeat(5000);

await worker.handle(generateTask({ title: longTitle }));

const savedEvent = await eventsCollection.findOne({});

/**
* 400 chars + ellipsis
*/
expect(savedEvent.payload.title.length).toBe(401);
expect(savedEvent.payload.title.endsWith('…')).toBe(true);
});

test('Should increment total events count on each processing', async () => {
await worker.handle(generateTask());
await worker.handle(generateTask());
Expand Down
Loading