diff --git a/lib/utils.js b/lib/utils.js index c28b9cb2..5a948859 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,4 +1,3 @@ -const _ = require('lodash'); const https = require('https'); /** @@ -9,159 +8,6 @@ const https = require('https'); */ module.exports.sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)); -/** - * Make `deepDiff` exportable - * return {object} - */ -module.exports.deepDiff = deepDiff; - -/** - * Make `deepMerge` exportable - * return {object} - */ -module.exports.deepMerge = deepMerge; - -/** - * Recursively scans two variables and returns another object with diffs - * - * @param {object|Array} source - source object - * @param {object|Array} target - target object - * @returns {object} - */ -function deepDiff(source, target) { - const sourceType = typeOf(source); - const targetType = typeOf(target); - - /** - * If we'll compare NOTHING with SOMETHING, the diff will be SOMETHING - */ - if (source === undefined) { - return target; - } - - /** - * If we'll compare SOMETHING with NOTHING, the diff will be NOTHING - */ - if (targetType === undefined) { - return undefined; - } - - /** - * We CAN'T compare apples with dogs - */ - if (sourceType !== targetType) { - return undefined; - } - - if (targetType === 'array') { - return arrayDiff(source, target); - } else if (targetType === 'object') { - return objectDiff(source, target); - } else if (source !== target) { - return target; - } else { - return source; - } -} - -/** - * Returns two arrays difference as an new array - * - * @param {Array} source - source object - * @param {Array} target - target object - * @returns {Array} - */ -function arrayDiff(source, target) { - const diffArray = []; - - for (let i = 0; i < target.length; i++) { - diffArray[i] = deepDiff(source[i], target[i]); - } - - return diffArray; -} - -/** - * Returns two objects difference as new object - * - * @param {object} objectA - first object for comparing - * @param {object} objectB - second object for comparing - * - * @returns {object} - */ -function objectDiff(objectA, objectB) { - const diffObject = {}; - - /** - * objectA is a subject, - * we compare objectB patches - * - * For that we enumerate objectB props and assume that - * target object has any changes - * - * But target object might have additional patches that might not be in subject - * This corner case says us that whole property is a patch - */ - if (!objectA) { - return objectB; - } - - Object.keys(objectB).forEach((prop) => { - const objectAItem = objectA[prop]; - const objectBItem = objectB[prop]; - - if (objectAItem === undefined) { - diffObject[prop] = objectBItem; - - return; - } - - if (objectAItem === objectBItem) { - return; - } - - diffObject[prop] = deepDiff(objectAItem, objectBItem); - }); - - return diffObject; -} - -/** - * Merge to objects recursively - * - * @param {object} target - target object - * @param {object[]} sources - sources for mering - * @returns {object} - */ -function deepMerge(target, ...sources) { - const isObject = (item) => item && typeOf(item) === 'object'; - - return _.mergeWith({}, target, ...sources, function (_subject, _target) { - if (_.isArray(_subject) && _.isArray(_target)) { - const biggerArray = _subject.length > _target.length ? _subject : _target; - const lesser = _subject.length > _target.length ? _target : _subject; - - return biggerArray.map((el, i) => { - if (isObject(el) && isObject(lesser[i])) { - return _.mergeWith({}, el, lesser[i]); - } else { - return el; - } - }); - } - }); -} - -/** - * Returns real type of passed variable - * - * @param {*} obj - value to check - * @returns {string} - */ -function typeOf(obj) { - return Object.prototype.toString.call(obj).match(/\s([a-zA-Z]+)/)[1].toLowerCase(); -} - /** * Sends alert to the Slack/Telegram * diff --git a/lib/utils.test.js b/lib/utils.test.js deleted file mode 100644 index fa2b096f..00000000 --- a/lib/utils.test.js +++ /dev/null @@ -1,273 +0,0 @@ -const utils = require('./utils'); - -describe('Utils', () => { - const dataProvider = [ - { - sourceObject: { - a: 3, - d: 1, - b: { - c: { - d: 6, - e: [], - }, - }, - }, - targetObject: { - a: 2, - b: { - c: { - d: 5, - e: [1, 1, 2], - }, - }, - }, - expectedDiff: { - a: 2, - b: { - c: { - d: 5, - e: [1, 1, 2], - }, - }, - }, - expectedMerge: { - a: 2, - d: 1, - b: { - c: { - d: 5, - e: [1, 1, 2], - }, - }, - }, - }, - { - sourceObject: { - a: 3, - d: 1, - b: { - c: { - d: 6, - e: [], - }, - }, - }, - targetObject: { - a: 3, - b: { - c: { - d: 6, - e: [], - }, - }, - }, - expectedDiff: { - b: { - c: { - e: [], - }, - }, - }, - expectedMerge: { - a: 3, - d: 1, - b: { - c: { - d: 6, - e: [], - }, - }, - }, - }, - /** - * First and Second object has array-property with different number of items - * First has less items count. - */ - - { - sourceObject: { - files: [ { line: 1 }, { line: 2 } ], - }, - targetObject: { - files: [ { line: 1 }, { line: 2 }, { line: 3 } ], - }, - expectedDiff: { - files: [ {}, {}, { line: 3 } ], - }, - expectedMerge: { - files: [ { line: 1 }, { line: 2 }, { line: 3 } ], - }, - }, - /** - * First and Second object has array-property with different number of items - * First has more items count. - */ - { - sourceObject: { - files: [ { line: 1 }, { line: 2 }, { line: 3 } ], - }, - targetObject: { - files: [ { line: 1 }, { line: 2 } ], - }, - expectedDiff: { - files: [ {}, {} ], - }, - expectedMerge: { - files: [ { line: 1 }, { line: 2 }, { line: 3 } ], - }, - }, - - /** - * The first - an empty array - * The second - array with the children non-empty array - */ - { - sourceObject: { - prop: [], - }, - targetObject: { - prop: [ [ 'i am not empty' ] ], - }, - expectedDiff: { - prop: [ [ 'i am not empty' ] ], - }, - expectedMerge: { - prop: [ [ 'i am not empty' ] ], - }, - }, - - /** - * Trying to compare two things of different type - */ - { - sourceObject: [], - targetObject: {}, - expectedDiff: undefined, - expectedMerge: {}, - }, - ]; - - test('should return right object diff', () => { - dataProvider.forEach((testCase) => { - const diff = utils.deepDiff(testCase.sourceObject, testCase.targetObject); - - expect(diff).toEqual(testCase.expectedDiff); - }); - }); - - test('should return right object merge', () => { - dataProvider.forEach((testCase) => { - const diff = utils.deepDiff(testCase.sourceObject, testCase.targetObject); - const merge = utils.deepMerge(testCase.sourceObject, diff); - - expect(merge).toEqual(testCase.expectedMerge); - }); - }); - - /** - * This test is a temporary solution to handle case with invalid events format sent by the PHP catcher - * - * The problem: the PHP catcher sends some data via incompatible format under invalid names and types. - * Those fields leads to the issues with diff calculation since fields could have different structure and types. - * - * The solution: deepDiff will return undefined in case of comparison of things with different types - * - * Original issue: - * https://github.com/codex-team/hawk.workers/issues/312 - * - * PHP Catcher issue: - * https://github.com/codex-team/hawk.php/issues/39 - */ - test('should not throw error comparing events with incompatible format', () => { - const originalStackTrace = [ - { - function: 'postAddComment', - class: 'Components\\Comments\\Comments', - object: {}, - type: '->', - args: [ - 286131, - {}, - ], - }, - ]; - - const repetitionStackTrace = [ - { - file: '/var/www/osnova/vendor/php-di/invoker/src/Invoker.php', - line: 74, - function: 'call_user_func_array', - args: [ - [ - {}, - 'sendWebhooksJob', - ], - [ - [ - { - id: 6697, - token: '539506', - event: 'new_comment', - url: 'https://callback.angry.space/vc_callback?date=2020&code=lMzBjOWZh@DADAD@jFlZmUy', - filter: '[]', - data: '[]', - removed: false, - }, - ], - { - type: 'new_comment', - data: { - id: 3206086, - url: 'https://somesite.io/trade/286961', - text: 'Это только со стороны так вроде долго, а если смотреть изнутри, то пока там освободиться купьер, пока найдут товар пока разберуться куда везти может и 4 часа пройти.', - media: [], - date: '2021-08-27T18:08:30+03:00', - creator: { - id: 27823, - avatar: 'https://s3.somesite.io/8ddee2e8-28e4-7863-425e-dd9b06deae5d/', - name: 'John S.', - url: 'https://somesite.io/u/27823-john-s', - }, - content: { - id: 286961, - title: 'Wildberries запустил доставку товаров за 2 часа в Петербурге', - url: 'https://somesite.io/trade/286961', - owner: { - id: 199122, - name: 'Торговля', - avatar: 'https://leonardo.osnova.io/d8fbb348-a8fd-641c-55dd-6a404055b457/', - url: 'https://somesite.io/trade', - }, - }, - replyTo: { - id: 3205883, - url: 'https://somesite.io/trade/286961', - text: 'Никто не пошутил, тогда это сделаю я!\n\n- 2.. часа!!1', - media: [], - creator: { - id: 877711, - avatar: 'https://leonardo.osnova.io/476a4e2c-8045-5b77-8a37-f6b1eb58bf93/', - name: 'Вадим Осадчий', - url: 'https://somesite.io/u/877711-john-doe', - }, - }, - }, - }, - ], - ], - }, - ]; - - const diff = utils.deepDiff(originalStackTrace, repetitionStackTrace); - - expect(diff).toEqual([ - { - file: '/var/www/osnova/vendor/php-di/invoker/src/Invoker.php', - line: 74, - function: 'call_user_func_array', - args: [undefined, undefined], - }, - ]); - }); -}); diff --git a/lib/utils/unsafeFields.ts b/lib/utils/unsafeFields.ts index 224292d4..f74564ea 100644 --- a/lib/utils/unsafeFields.ts +++ b/lib/utils/unsafeFields.ts @@ -1,4 +1,6 @@ -import { GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; +import { GroupedEventDBScheme, RepetitionDBScheme as RepetitionDBSchemeType } from '@hawk.so/types'; + +type RepetitionDBScheme = Omit & Partial>; /** * Fields in event payload with unsafe data for encoding before saving in database @@ -14,12 +16,24 @@ export const unsafeFields = ['context', 'addons'] as const; export function decodeUnsafeFields(event: GroupedEventDBScheme | RepetitionDBScheme): void { unsafeFields.forEach((field) => { try { - const fieldValue = event.payload[field]; + let fieldValue: unknown; + + if ('delta' in event) { + fieldValue = event.delta[field]; + } else { + fieldValue = event.payload[field]; + } if (typeof fieldValue === 'string') { - event.payload[field] = JSON.parse(fieldValue); + if ('delta' in event) { + event.delta[field] = JSON.parse(fieldValue); + } else { + event.payload[field] = JSON.parse(fieldValue); + } } - } catch { /* ignore if caught */ } + } catch { + console.error(`Failed to parse field ${field} in event ${event._id}`); + } }); } @@ -30,7 +44,24 @@ export function decodeUnsafeFields(event: GroupedEventDBScheme | RepetitionDBSch */ export function encodeUnsafeFields(event: GroupedEventDBScheme | RepetitionDBScheme): void { unsafeFields.forEach((field) => { - const fieldValue = event.payload[field]; + let fieldValue: unknown; + + /** + * Repetition includes delta field, grouped event includes payload + */ + if ('delta' in event) { + fieldValue = event.delta[field]; + } else { + fieldValue = event.payload[field]; + } + + /** + * Repetition diff can omit these fields if they are not changed + */ + if (fieldValue === undefined) { + return; + } + let newValue: string; try { @@ -38,8 +69,17 @@ export function encodeUnsafeFields(event: GroupedEventDBScheme | RepetitionDBSch newValue = JSON.stringify(fieldValue); } } catch { + console.error(`Failed to stringify field ${field} in event ${event._id}`); newValue = undefined; } - event.payload[field] = newValue; + + /** + * Repetition includes delta field, grouped event includes payload + */ + if ('delta' in event) { + event.delta[field] = newValue; + } else { + event.payload[field] = newValue; + } }); } diff --git a/package.json b/package.json index 174f356a..ca3530ba 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:slack": "jest workers/slack", "test:limiter": "jest workers/limiter", "test:grouper": "jest workers/grouper", + "test:diff": "jest ./workers/grouper/tests/diff.test.ts", "test:paymaster": "jest workers/paymaster", "test:notifier": "jest workers/notifier", "test:js": "jest workers/javascript", @@ -56,7 +57,6 @@ "codex-accounting-sdk": "codex-team/codex-accounting-sdk", "debug": "^4.1.1", "dotenv": "^8.2.0", - "lodash.mergewith": "^4.6.2", "migrate-mongo": "^7.2.1", "mockdate": "^3.0.2", "mongodb": "^3.5.7", diff --git a/workers/grouper/package.json b/workers/grouper/package.json index 4b376934..2703c06d 100644 --- a/workers/grouper/package.json +++ b/workers/grouper/package.json @@ -10,6 +10,7 @@ "workerType": "grouper", "dependencies": { "@types/redis": "^2.8.28", + "@n1ru4l/json-patch-plus": "^0.2.0", "js-levenshtein": "^1.1.6" } } diff --git a/workers/grouper/src/data-filter.ts b/workers/grouper/src/data-filter.ts index 243cdd7f..51fef51c 100644 --- a/workers/grouper/src/data-filter.ts +++ b/workers/grouper/src/data-filter.ts @@ -1,4 +1,4 @@ -import { EventAddons, EventDataAccepted } from '@hawk.so/types'; +import type { EventAddons, EventDataAccepted } from '@hawk.so/types'; import { unsafeFields } from '../../../lib/utils/unsafeFields'; /** diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 1c47c7b5..47b1b1dc 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -2,12 +2,12 @@ import './env'; import * as crypto from 'crypto'; import * as mongodb from 'mongodb'; import { DatabaseController } from '../../../lib/db/controller'; -import * as utils from '../../../lib/utils'; import { Worker } from '../../../lib/worker'; import * as WorkerNames from '../../../lib/workerNames'; import * as pkg from '../package.json'; -import { GroupWorkerTask } from '../types/group-worker-task'; -import { EventAddons, EventDataAccepted, GroupedEventDBScheme, RepetitionDBScheme } from '@hawk.so/types'; +import type { GroupWorkerTask, RepetitionDelta } from '../types/group-worker-task'; +import type { EventAddons, EventDataAccepted, GroupedEventDBScheme } from '@hawk.so/types'; +import type { RepetitionDBScheme } from '../types/repetition'; import { DatabaseReadWriteError, DiffCalculationError, ValidationError } from '../../../lib/workerErrors'; import { decodeUnsafeFields, encodeUnsafeFields } from '../../../lib/utils/unsafeFields'; import HawkCatcher from '@hawk.so/nodejs'; @@ -15,6 +15,7 @@ import { MS_IN_SEC } from '../../../lib/utils/consts'; import DataFilter from './data-filter'; import RedisHelper from './redisHelper'; import levenshtein from 'js-levenshtein'; +import { computeDelta } from './utils/repetitionDiff'; import TimeMs from '../../../lib/utils/time'; /** @@ -181,20 +182,22 @@ export default class GrouperWorker extends Worker { */ decodeUnsafeFields(existedEvent); - let diff; + let delta: RepetitionDelta; try { /** - * Save event's repetitions + * Calculate delta between original event and repetition */ - diff = utils.deepDiff(existedEvent.payload, task.event); + delta = computeDelta(existedEvent.payload, task.event); } catch (e) { + console.error(e); throw new DiffCalculationError(e, existedEvent.payload, task.event); } const newRepetition = { groupHash: uniqueEventHash, - payload: diff, + delta: JSON.stringify(delta), + timestamp: task.event.timestamp, } as RepetitionDBScheme; repetitionId = await this.saveRepetition(task.projectId, newRepetition); @@ -340,7 +343,7 @@ export default class GrouperWorker extends Worker { * @param count - how many events to return * @returns {GroupedEventDBScheme[]} list of the last N unique events */ - private findLastEvents(projectId: string, count): Promise { + private findLastEvents(projectId: string, count: number): Promise { return this.cache.get(`last:${count}:eventsOf:${projectId}`, async () => { return this.eventsDb.getConnection() .collection(`events:${projectId}`) @@ -495,7 +498,7 @@ export default class GrouperWorker extends Worker { * @returns {string} cache key for event */ private async getEventCacheKey(projectId: string, groupHash: string): Promise { - return `${projectId}:${JSON.stringify({ groupHash: groupHash })}`; + return `${projectId}:${JSON.stringify({ groupHash })}`; } /** diff --git a/workers/grouper/src/redisHelper.ts b/workers/grouper/src/redisHelper.ts index 391389b5..a655c24b 100644 --- a/workers/grouper/src/redisHelper.ts +++ b/workers/grouper/src/redisHelper.ts @@ -1,5 +1,6 @@ import HawkCatcher from '@hawk.so/nodejs'; -import { createClient, RedisClientType } from 'redis'; +import type { RedisClientType } from 'redis'; +import { createClient } from 'redis'; import createLogger from '../../../lib/logger'; /** diff --git a/workers/grouper/src/utils/repetitionDiff.ts b/workers/grouper/src/utils/repetitionDiff.ts new file mode 100644 index 00000000..99ae4838 --- /dev/null +++ b/workers/grouper/src/utils/repetitionDiff.ts @@ -0,0 +1,19 @@ +import type { EventAddons, EventDataAccepted } from '@hawk.so/types'; +import { diff } from '@n1ru4l/json-patch-plus'; +import type { RepetitionDelta } from '../../types/group-worker-task'; + +/** + * Calculate delta between original event and repetition + * + * @param originalEvent - first event + * @param repetition - one of remaining events + * @returns delta {RepetitionDelta} + */ +export function computeDelta(originalEvent: EventDataAccepted, repetition: EventDataAccepted): RepetitionDelta { + const delta = diff({ + left: originalEvent, + right: repetition, + }); + + return delta; +} diff --git a/workers/grouper/tests/data-filter.test.ts b/workers/grouper/tests/data-filter.test.ts index 556233b2..15c8d4a8 100644 --- a/workers/grouper/tests/data-filter.test.ts +++ b/workers/grouper/tests/data-filter.test.ts @@ -1,5 +1,5 @@ import '../../../env-test'; -import { EventAddons, EventDataAccepted, Json } from '@hawk.so/types'; +import type { EventAddons, EventDataAccepted, Json } from '@hawk.so/types'; import DataFilter from '../src/data-filter'; jest.mock('amqplib'); diff --git a/workers/grouper/tests/diff.test.ts b/workers/grouper/tests/diff.test.ts new file mode 100644 index 00000000..c636add0 --- /dev/null +++ b/workers/grouper/tests/diff.test.ts @@ -0,0 +1,127 @@ +import { patch } from '@n1ru4l/json-patch-plus'; +import '../../../env-test'; + +import { computeDelta } from '../src/utils/repetitionDiff'; +import { generateEvent } from './mocks/generateEvent'; + +/** + * Check that we always can get the whole repetition by applying delta to the original event + */ +describe('Diff', () => { + it('Original event has backtrace, repetition has no backtrace, merged event has no backtrace', () => { + const originalEvent = generateEvent(); + + const repetition = generateEvent({ + backtrace: null, + context: { + testField: 10, + }, + }); + + const delta = computeDelta(originalEvent, repetition); + + const patched = patch({ + left: originalEvent, + delta, + }); + + expect(patched).toEqual(repetition); + }); + it('Original event has no backtrace, repetition has backtrace, merged event has backtrace', () => { + const originalEvent = generateEvent({ + backtrace: null, + context: { + testField: 10, + }, + }); + + const repetition = generateEvent({ + backtrace: [ + { + file: 'test.ts', + line: 10, + }, + ], + }); + + const delta = computeDelta(originalEvent, repetition); + + const patched = patch({ + left: originalEvent, + delta, + }); + + expect(patched).toEqual(repetition); + }); + + it('Original event and repetition have different backtrace', () => { + const originalEvent = generateEvent({ + backtrace: [ + { + file: 'test.ts', + line: 10, + }, + ], + }); + + const repetition = generateEvent({ + backtrace: [ + { + file: 'test.ts', + line: 11, + }, + ], + }); + + const delta = computeDelta(originalEvent, repetition); + + const patched = patch({ + left: originalEvent, + delta, + }); + + expect(patched.backtrace).toEqual(repetition.backtrace); + }); + + it('Original event has context with "someField" and repetition has no "someField"', () => { + const originalEvent = generateEvent({ + context: { + someField: 'someValue', + }, + }); + + const repetition = generateEvent({ + context: {}, + }); + + const delta = computeDelta(originalEvent, repetition); + + const patched = patch({ + left: originalEvent, + delta, + }); + + expect(patched.context).toEqual(repetition.context); + }); + + it('Original event has no context with "someField" and repetition has "someField"', () => { + const originalEvent = generateEvent({ + context: {}, + }); + + const repetition = generateEvent({ + context: { + someField: 'someValue', + }, + }); + + const delta = computeDelta(originalEvent, repetition); + + const patched = patch({ + left: originalEvent, + delta, + }); + + expect(patched.context).toEqual(repetition.context); + }); +}); diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index 526e243b..fcbb9554 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -1,11 +1,14 @@ import '../../../env-test'; import GrouperWorker from '../src'; -import { GroupWorkerTask } from '../types/group-worker-task'; -import { createClient, RedisClientType } from 'redis'; -import { Collection, MongoClient } from 'mongodb'; -import { EventAddons, EventDataAccepted } from '@hawk.so/types'; +import type { GroupWorkerTask, RepetitionDelta } from '../types/group-worker-task'; +import type { RedisClientType } from 'redis'; +import { createClient } from 'redis'; +import type { Collection } from 'mongodb'; +import { MongoClient } from 'mongodb'; +import type { EventAddons, EventDataAccepted } from '@hawk.so/types'; import { MS_IN_SEC } from '../../../lib/utils/consts'; import * as mongodb from 'mongodb'; +import { patch } from '@n1ru4l/json-patch-plus'; jest.mock('amqplib'); @@ -186,10 +189,10 @@ describe('GrouperWorker', () => { }); test('Should increment usersAffected count if users are different', async () => { - await worker.handle(generateTask()); - await worker.handle(generateTask()); - await worker.handle(generateTask()); - await worker.handle(generateTask()); + await worker.handle(generateTask({ user: { id: generateRandomId() } })); + await worker.handle(generateTask({ user: { id: generateRandomId() } })); + await worker.handle(generateTask({ user: { id: generateRandomId() } })); + await worker.handle(generateTask({ user: { id: generateRandomId() } })); expect((await eventsCollection.findOne({})).usersAffected).toBe(4); expect((await dailyEventsCollection.findOne({})).affectedUsers).toBe(4); @@ -317,8 +320,10 @@ describe('GrouperWorker', () => { test('Should stringify payload`s addons and context fields', async () => { await worker.handle(generateTask()); - expect(typeof (await eventsCollection.findOne({})).payload.addons).toBe('string'); - expect(typeof (await eventsCollection.findOne({})).payload.context).toBe('string'); + const savedEvent = await eventsCollection.findOne({}); + + expect(typeof savedEvent.payload.addons).toBe('string'); + expect(typeof savedEvent.payload.context).toBe('string'); }); test('Should save event even if its context is type of string', async () => { @@ -372,7 +377,7 @@ describe('GrouperWorker', () => { }).toArray()).length).toBe(2); }); - test('Should stringify payload`s addons and context fields', async () => { + test('Should stringify delta', async () => { const generatedTask = generateTask(); await worker.handle(generateTask()); @@ -381,29 +386,43 @@ describe('GrouperWorker', () => { event: { ...generatedTask.event, addons: { test: '8fred' }, + context: { test: '8fred' }, }, }); const savedRepetition = await repetitionsCollection.findOne({}); - expect(typeof savedRepetition.payload.addons).toBe('string'); - expect(typeof savedRepetition.payload.context).toBe('string'); + expect(typeof savedRepetition.delta).toBe('string'); }); test('Should correctly calculate diff after encoding original event when they are the same', async () => { - await worker.handle(generateTask()); - await worker.handle(generateTask()); + await worker.handle(generateTask({ user: { id: '123' } })); + await worker.handle(generateTask({ user: { id: '123' } })); - expect((await repetitionsCollection.findOne({})).payload.context).toBe('{}'); + const savedRepetition = await repetitionsCollection.findOne({}); + + const savedDelta = savedRepetition.delta as string; + const parsedDelta = JSON.parse(savedDelta) as RepetitionDelta; + + expect(parsedDelta.type).toBe(undefined); + expect(parsedDelta.backtrace).toBe(undefined); + expect(parsedDelta.context).toBe(undefined); + expect(parsedDelta.addons).toBe(undefined); + expect(parsedDelta.release).toBe(undefined); + expect(parsedDelta.user).toBe(undefined); + expect(parsedDelta.catcherVersion).toBe(undefined); /** - * Should to be true when bug in utils.deepDiff will be fixed + * Timestamp always unique, so it should be present in a stored payload diff */ - // expect((await repetitionsCollection.findOne({})).payload.addons).toBe('{}'); + expect(parsedDelta.timestamp).not.toBe(undefined); + expect(typeof parsedDelta.timestamp).toBe('object'); }); test('Should correctly calculate diff after encoding original event when they are different', async () => { - await worker.handle(generateTask()); + const originalGeneratedEvent = generateTask(); + + await worker.handle(originalGeneratedEvent); const generatedTask = generateTask(); @@ -422,8 +441,33 @@ describe('GrouperWorker', () => { }, }); - expect((await repetitionsCollection.findOne({})).payload.context).toBe('{"testField":9}'); - expect((await repetitionsCollection.findOne({})).payload.addons).toBe('{"vue":{"props":{"test-test":true}}}'); + const savedEvent = await eventsCollection.findOne({}); + const savedRepetition = await repetitionsCollection.findOne({}); + const savedDelta = savedRepetition.delta as string; + const parsedDelta = JSON.parse(savedDelta) as RepetitionDelta; + + /** + * Parse context and addons from string to object + */ + savedEvent.payload.context = JSON.parse(savedEvent.payload.context); + savedEvent.payload.addons = JSON.parse(savedEvent.payload.addons); + + expect(typeof parsedDelta.context).toBe('object'); + expect(typeof parsedDelta.addons).toBe('object'); + + const patched = patch({ + left: savedEvent.payload, + delta: parsedDelta, + }); + + expect(patched.context).toEqual({ + testField: 9, + }); + expect(patched.addons).toEqual({ + vue: { + props: { 'test-test': true }, + }, + }); }); }); diff --git a/workers/grouper/tests/mocks/generateEvent.ts b/workers/grouper/tests/mocks/generateEvent.ts new file mode 100644 index 00000000..a89a4d54 --- /dev/null +++ b/workers/grouper/tests/mocks/generateEvent.ts @@ -0,0 +1,36 @@ +import type { EventAddons, EventDataAccepted } from '@hawk.so/types'; +import { generateRandomId } from './randomId'; + +/** + * Mocked User id used for tests + */ +const userIdMock = generateRandomId(); + +/** + * Generate mocked event + * + * @param event - Partial event data to override default values + */ +export function generateEvent(event: Partial> = undefined): EventDataAccepted { + return { + title: 'Hawk client catcher test', + timestamp: (new Date()).getTime(), + backtrace: [], + user: { + id: userIdMock, + }, + context: { + testField: 8, + 'ima$ge.jpg': 'img', + }, + addons: { + vue: { + props: { + 'test-test': false, + 'ima$ge.jpg': 'img', + }, + }, + }, + ...event, + }; +} diff --git a/workers/grouper/tests/mocks/generateTask.ts b/workers/grouper/tests/mocks/generateTask.ts new file mode 100644 index 00000000..c92ed3ad --- /dev/null +++ b/workers/grouper/tests/mocks/generateTask.ts @@ -0,0 +1,19 @@ +import type { EventAddons, EventDataAccepted } from '@hawk.so/types'; +import type { GroupWorkerTask } from '../../types/group-worker-task'; +import { projectIdMock } from './projectId'; +import { generateEvent } from './generateEvent'; + +/** + * Generates task for testing + * + * @param event - allows to override some event properties in generated task + */ +export function generateTask( + event: Partial> = undefined +): GroupWorkerTask { + return { + projectId: projectIdMock, + catcherType: 'grouper', + event: generateEvent(event), + }; +} diff --git a/workers/grouper/tests/mocks/projectId.ts b/workers/grouper/tests/mocks/projectId.ts new file mode 100644 index 00000000..1ec4989e --- /dev/null +++ b/workers/grouper/tests/mocks/projectId.ts @@ -0,0 +1,4 @@ +/** + * Mocked Project id used for tests + */ +export const projectIdMock = '5d206f7f9aaf7c0071d64596'; diff --git a/workers/grouper/tests/mocks/randomId.ts b/workers/grouper/tests/mocks/randomId.ts new file mode 100644 index 00000000..cefd50df --- /dev/null +++ b/workers/grouper/tests/mocks/randomId.ts @@ -0,0 +1,12 @@ +/** + * Returns random string + */ +export function generateRandomId(): string { + const FIRST_RANDOM_START = 2; + const FIRST_RANDOM_END = 15; + const RADIX = 36; + + return Math.random().toString(RADIX) + .substring(FIRST_RANDOM_START, FIRST_RANDOM_END) + Math.random().toString(RADIX) + .substring(FIRST_RANDOM_START, FIRST_RANDOM_END); +} diff --git a/workers/grouper/types/group-worker-task.d.ts b/workers/grouper/types/group-worker-task.d.ts index 76c34c50..31498356 100644 --- a/workers/grouper/types/group-worker-task.d.ts +++ b/workers/grouper/types/group-worker-task.d.ts @@ -1,5 +1,6 @@ -import { EventDataAccepted, EventAddons } from '@hawk.so/types'; -import { WorkerTask } from '../../../lib/types/worker-task'; +import type { EventDataAccepted, EventAddons } from '@hawk.so/types'; +import type { WorkerTask } from '../../../lib/types/worker-task'; +import type { Delta } from '@n1ru4l/json-patch-plus'; /** * Language-workers adds tasks for Group Worker in this format. @@ -21,3 +22,8 @@ export interface GroupWorkerTask extends WorkerTask { */ event: EventDataAccepted; } + +/** + * Delta of the original event and the repetition + */ +export type RepetitionDelta = Delta; diff --git a/workers/grouper/types/repetition.ts b/workers/grouper/types/repetition.ts new file mode 100644 index 00000000..8f0eb7cb --- /dev/null +++ b/workers/grouper/types/repetition.ts @@ -0,0 +1,3 @@ +import type { RepetitionDBScheme as RepetitionDBSchemeType } from '@hawk.so/types'; + +export type RepetitionDBScheme = Omit & Partial>; diff --git a/yarn.lock b/yarn.lock index c83d5fac..6de3c045 100644 --- a/yarn.lock +++ b/yarn.lock @@ -689,6 +689,11 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@n1ru4l/json-patch-plus@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@n1ru4l/json-patch-plus/-/json-patch-plus-0.2.0.tgz#b8fa09fd980c3460dfdc109a7c4cc5590157aa6b" + integrity sha512-pLkJy83/rVfDTyQgDSC8GeXAHEdXNHGNJrB1b7wAyGQu0iv7tpMXntKVSqj0+XKNVQbco40SZffNfVALzIt0SQ== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" @@ -4886,11 +4891,6 @@ lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== -lodash.mergewith@^4.6.2: - version "4.6.2" - resolved "https://registry.yarnpkg.com/lodash.mergewith/-/lodash.mergewith-4.6.2.tgz#617121f89ac55f59047c7aec1ccd6654c6590f55" - integrity sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ== - lodash.omit@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.omit/-/lodash.omit-4.5.0.tgz#6eb19ae5a1ee1dd9df0b969e66ce0b7fa30b5e60"