Merge branch 'master' of github.com:codex-team/hawk.workers into new-diff

Author: slaveeks

lib/worker.ts

@@ -327,10 +327,8 @@ export abstract class Worker {
         };
       }
 
-
       console.log('handle error');
       console.log(e);
-      
 
       HawkCatcher.send(e, context);
 

package.json

@@ -16,6 +16,7 @@
         "migrate:down": "migrate-mongo down",
         "lint": "eslint -c ./.eslintrc.js --ext .ts,.js --fix .",
         "test": "jest --coverage --runInBand",
+        "test:archiver": "jest workers/archiver",
         "test:base": "jest lib/worker",
         "test:db": "jest lib/db/",
         "test:cache": "jest lib/cache",
@@ -29,6 +30,7 @@
         "test:diff": "jest ./workers/grouper/tests/diff.test.ts",
         "test:paymaster": "jest workers/paymaster",
         "test:notifier": "jest workers/notifier",
+        "test:js": "jest workers/javascript",
         "test:clear": "jest --clearCache",
         "run-default": "yarn worker hawk-worker-default",
         "run-sentry": "yarn worker hawk-worker-sentry",
@@ -69,7 +71,9 @@
         "typescript": "^4.9.4",
         "uuid": "^8.3.0",
         "winston": "^3.2.1",
-        "yup": "^0.28.5"
+        "yup": "^0.28.5",
+        "@babel/parser": "^7.26.9",
+        "@babel/traverse": "7.26.9"
     },
     "devDependencies": {
         "@shelf/jest-mongodb": "^1.2.3",

workers/archiver/src/index.ts

@@ -2,7 +2,7 @@ import { DatabaseController } from '../../../lib/db/controller';
 import { Worker } from '../../../lib/worker';
 import * as pkg from '../package.json';
 import asyncForEach from '../../../lib/utils/asyncForEach';
-import { Collection, Db, GridFSBucket, ObjectId } from 'mongodb';
+import { Collection, Db, GridFSBucket, ObjectId, ObjectID } from 'mongodb';
 import axios from 'axios';
 import { ReleaseFileData, ReleaseRecord, ReportData, ReportDataByProject } from './types';
 import * as path from 'path';
@@ -348,13 +348,25 @@ export default class ArchiverWorker extends Worker {
    * @param project - project to handle
    */
   private async removeOldReleases(project: ProjectDBScheme): Promise<number> {
-    const RELEASES_COUNT_TO_REMOVE = 3;
+    const RELEASES_COUNT_TO_STAY = 2;
+    const maxDaysInSeconds = +process.env.MAX_DAYS_NUMBER * HOURS_IN_DAY * MINUTES_IN_HOUR * SECONDS_IN_MINUTE;
+    /**
+     * Create timestamp for one month ago
+     */
+    const maxOldTimestamp = (new Date().getTime()) / MS_IN_SEC - maxDaysInSeconds;
+
+    /**
+     * Create mongo ObjectId for record that was inserted one month ago
+     */
+    const objectIdThreshold = ObjectID.createFromTime(maxOldTimestamp);
+
     const releasesToRemove = await this.releasesCollection
       .find({
         projectId: project._id.toString(),
+        _id: { $lt: objectIdThreshold },
       })
       .sort({ _id: -1 })
-      .skip(RELEASES_COUNT_TO_REMOVE)
+      .skip(RELEASES_COUNT_TO_STAY)
       .toArray();
 
     const filesToDelete = releasesToRemove.reduce<ReleaseFileData[]>(

workers/archiver/tests/index.test.ts

@@ -51,6 +51,10 @@ describe('Archiver worker', () => {
     await eventsCollection.insertMany(mockedEvents);
   });
 
+  beforeEach(async () => {
+    await db.collection('releases').deleteMany({});
+  })
+
   test('Should correctly remove old events', async () => {
     /**
      * Worker initialization
@@ -111,6 +115,22 @@ describe('Archiver worker', () => {
   test('Should remove old releases', async () => {
     await db.collection('releases').insertMany(mockedReleases);
 
+    const mockedReleasesLength = mockedReleases.length;
+
+    const releasesToStay = {
+      _id: new ObjectId(),
+      projectId: '5e4ff518628a6c714515f4da',
+      release: 'releasetostay',
+      files: [ {
+        _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
+      } ],
+    };
+
+    /**
+     * Insert one release with object id based on current time, it should not be removed
+     */
+    await db.collection('releases').insert(releasesToStay)
+
     const worker = new ArchiverWorker();
 
     await worker.start();
@@ -122,11 +142,41 @@ describe('Archiver worker', () => {
       .find({})
       .toArray();
 
-    expect(newReleasesCollection).toEqual(mockedReleases.slice(mockedReleases.length - 3));
-    expect(gridFsDeleteMock).toHaveBeenCalledTimes(mockedReleases.length - 3);
+    expect(newReleasesCollection).toEqual([
+      mockedReleases[mockedReleasesLength - 2],
+      mockedReleases[mockedReleasesLength - 1],
+      releasesToStay,
+    ]);
+
+    expect(gridFsDeleteMock).toHaveBeenCalledTimes(mockedReleases.length - 2);
     await worker.finish();
   });
 
+  test('Should leave two releases if all of the releases are more than month old', async () => {
+    /**
+     * Clear collection after previous test
+     */
+    await db.collection('releases').insertMany(mockedReleases);
+
+    const mockedReleasesLength = mockedReleases.length;
+
+    const worker = new ArchiverWorker();
+
+    await worker.start();
+
+    await worker['removeOldReleases'](mockedProject);
+
+    const newReleasesCollection = await db.collection('releases')
+      .find({})
+      .toArray();
+
+    expect(newReleasesCollection).toEqual([
+      mockedReleases[mockedReleasesLength - 2],
+      mockedReleases[mockedReleasesLength - 1],
+    ])
+    await worker.finish();
+  })
+
   afterAll(async () => {
     await db.dropCollection('releases');
     await db.dropCollection('projects');

workers/archiver/tests/releases.mock.ts

@@ -2,79 +2,79 @@ import { ObjectId } from 'mongodb';
 
 const mockedReleases = [
   {
-    _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
+    _id: new ObjectId('5e7119ec6570b9405cdc5b48'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'kjjцувgfchfc',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb119f16570b9405cdc5b4b'),
+    _id: new ObjectId('5e7119f16570b9405cdc5b4b'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'kwedewdc',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb119f56570b9405cdc5b4e'),
+    _id: new ObjectId('5e7119f56570b9405cdc5b4e'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'f43fwew',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb127776570b9405cdc5b51'),
+    _id: new ObjectId('5e7127776570b9405cdc5b51'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'f43fwerferfeferfew',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb1277b6570b9405cdc5b54'),
+    _id: new ObjectId('5e71277b6570b9405cdc5b54'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'f43fweerferfrferfeferfew',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb127816570b9405cdc5b57'),
+    _id: new ObjectId('5e7127816570b9405cdc5b57'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'f4erferg4few',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb1278a6570b9405cdc5b5a'),
+    _id: new ObjectId('5e71278a6570b9405cdc5b5a'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'ff34f34f34',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb127926570b9405cdc5b5d'),
+    _id: new ObjectId('5e7127926570b9405cdc5b5d'),
     projectId: '5e4ff518628a6c714515f4da',
     release: '34df34re',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb127ac6570b9405cdc5b60'),
+    _id: new ObjectId('5e7127ac6570b9405cdc5b60'),
     projectId: '5e4ff518628a6c714515f4da',
     release: 'wefwefweghbrvd',
     files: [ {
       _id: new ObjectId('5eb119ec6570b9405cdc5b48'),
     } ],
   },
   {
-    _id: new ObjectId('5eb127b56570b9405cdc5b63'),
+    _id: new ObjectId('5e7127b56570b9405cdc5b63'),
     projectId: '5e4ff518628a6c714515f4da',
     release: '45gfewd',
     files: [ {

workers/email/src/templates/extensions.ts

@@ -46,6 +46,7 @@ Twig.extendFilter('leftTrim', ((value: string, maxLen: number): string => {
   if (value.length > maxLen) {
     return '…' + value.slice(value.length - maxLen);
   }
+
   return value;
 }) as unknown as (value: any, params: false | any[]) => string); // tmp case. We need to check if TS says correct types or our implementation is correct
 

workers/grouper/src/index.ts

@@ -98,9 +98,7 @@ export default class GrouperWorker extends Worker {
     /**
      * Find event by group hash.
      */
-    let existedEvent = await this.getEvent(task.projectId, {
-      groupHash: uniqueEventHash,
-    });
+    let existedEvent = await this.getEvent(task.projectId, uniqueEventHash);
 
     /**
      * If we couldn't group by group hash (title), try grouping by Levenshtein distance with last N events
@@ -134,7 +132,7 @@ export default class GrouperWorker extends Worker {
 
     if (isFirstOccurrence) {
       try {
-        const incrementAffectedUsers = task.event.user ? true : false;
+        const incrementAffectedUsers = !!task.event.user;
 
         /**
          * Insert new event
@@ -147,6 +145,13 @@ export default class GrouperWorker extends Worker {
           usersAffected: incrementAffectedUsers ? 1 : 0,
         } as GroupedEventDBScheme);
 
+        const eventCacheKey = await this.getEventCacheKey(task.projectId, uniqueEventHash);
+
+        /**
+         * If event is saved, then cached event state is no longer actual, so we should remove it
+         */
+        this.cache.del(eventCacheKey);
+
         /**
          * Increment daily affected users for the first event
          */
@@ -158,7 +163,7 @@ export default class GrouperWorker extends Worker {
          */
         if (e.code?.toString() === DB_DUPLICATE_KEY_ERROR) {
           console.log('DB_DUPLICATE_KEY_ERROR');
-          
+
           HawkCatcher.send(new Error('[Grouper] MongoError: E11000 duplicate key error collection'));
           await this.handle(task);
 
@@ -275,7 +280,7 @@ export default class GrouperWorker extends Worker {
         })
         .limit(count)
         .toArray();
-    },  
+    },
     /**
      * TimeMs class stores time intervals in milliseconds, however NodeCache ttl needs to be specified in seconds
      */
@@ -300,7 +305,6 @@ export default class GrouperWorker extends Worker {
       return [false, false];
     }
 
-
     /**
      * Default to true - we'll set to false if conditions are met
      */
@@ -392,25 +396,37 @@ export default class GrouperWorker extends Worker {
    * Returns finds event by query from project with passed ID
    *
    * @param projectId - project's identifier
-   * @param query - mongo query string
+   * @param groupHash - group hash of the event   
    */
-  private async getEvent(projectId: string, query: Record<string, unknown>): Promise<GroupedEventDBScheme> {
+  private async getEvent(projectId: string, groupHash: string): Promise<GroupedEventDBScheme> {
     if (!mongodb.ObjectID.isValid(projectId)) {
       throw new ValidationError('Controller.saveEvent: Project ID is invalid or missed');
     }
 
-    const eventCacheKey = `${projectId}:${JSON.stringify(query)}`;
+    const eventCacheKey = await this.getEventCacheKey(projectId, groupHash);
 
     return this.cache.get(eventCacheKey, async () => {
       return this.db.getConnection()
         .collection(`events:${projectId}`)
-        .findOne(query)
+        .findOne({
+          groupHash,
+        })
         .catch((err) => {
           throw new DatabaseReadWriteError(err);
         });
     });
   }
 
+  /**
+   * Method that returns event cache key based on projectId and groupHash
+   * @param projectId - used for cache key creation
+   * @param groupHash - used for cache key creation
+   * @returns cache key
+   */
+  private async getEventCacheKey(projectId: string, groupHash: string): Promise<string> {
+    return `${projectId}:${JSON.stringify({groupHash: groupHash})}`
+  }
+
   /**
    * Save event to database
    *

workers/grouper/src/redisHelper.ts

@@ -40,7 +40,6 @@ export default class RedisHelper {
     } catch (error) {
       console.error('Error creating redis client', error);
     }
-
   }
 
   /**