Merge branch 'master' of github.com:codex-team/hawk.workers into new-diff

Author: slaveeks

.github/workflows/build-and-push-docker-image.yml

@@ -7,44 +7,36 @@ env:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: endsWith(github.ref, '/prod') || endsWith(github.ref, '/stage') || endsWith(github.ref, '/migration')
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Build and push production image
-        if: endsWith(github.ref, '/prod')
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: ./Dockerfile
-          tags: ${{ env.DOCKER_REPO }}:prod
-          push: ${{ endsWith(github.ref, '/prod') }}
-
-      - name: Build and push stage image
-        if: endsWith(github.ref, '/stage')
-        uses: docker/build-push-action@v2
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
         with:
-          context: .
-          file: ./stage.Dockerfile
-          tags: ${{ env.DOCKER_REPO }}:stage
-          push: ${{ endsWith(github.ref, '/stage') }}
+          images: ${{ env.DOCKER_REPO }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value={{branch}}-{{sha}}-{{date 'X'}},enable=${{ startsWith(github.ref, 'refs/heads') }}
 
-      - name: Build and push migration image
-        if: endsWith(github.ref, '/migration')
-        uses: docker/build-push-action@v2
+      - name: Build and push production image
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./stage.Dockerfile
-          tags: ${{ env.DOCKER_REPO }}:migration
-          push: ${{ endsWith(github.ref, '/migration') }}
+          file: ./Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          push: true

.github/workflows/check-build.yaml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2

.github/workflows/tests.yml

@@ -5,7 +5,7 @@ on: [push]
 jobs:
   lint:
     name: Unit testing
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js 16.x

.github/workflows/update-monorepository.yml

@@ -7,7 +7,7 @@ env:
   REPOSITORY_MONO_PATH: workers
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Mono repository update
         uses: peter-evans/repository-dispatch@v1

jest-mongodb-config.js

@@ -3,6 +3,8 @@ module.exports = {
     instance: {
       port: 55010,
       dbName: 'hawk',
+      replSet: 'rs0',
+      storageEngine: 'wiredTiger',
     },
     binary: {
       version: '6.0.2',

jest.config.js

@@ -27,5 +27,5 @@ module.exports = {
 
   setupFiles: [ './jest.setup.js' ],
 
-  setupFilesAfterEnv: [ './jest.setup.redis-mock.js' ],
+  setupFilesAfterEnv: ['./jest.setup.redis-mock.js', './jest.setup.mongo-repl-set.js'],
 };

jest.setup.mongo-repl-set.js

@@ -0,0 +1,50 @@
+const { MongoClient } = require('mongodb');
+
+let admin;
+let connection;
+
+beforeAll(async () => {
+  connection = await MongoClient.connect('mongodb://127.0.0.1:55010/hawk?', {
+    useNewUrlParser: true,
+    useUnifiedTopology: true,
+  });
+
+  admin = connection.db().admin();
+
+  try {
+    let status = await admin.command({ replSetGetStatus: 1 }).catch(() => null);
+
+    if (status && status.ok) {
+      console.log('✅ Replica set already initialized');
+    } else {
+      await admin.command({ replSetInitiate: {} });
+      console.log('✅ Replica set initiated');
+    }
+
+    const startTime = Date.now();
+    const timeout = 15000;
+
+    /**
+     * Wait for the replica set to initialize all nodes
+     */
+    do {
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      status = await admin.command({ replSetGetStatus: 1 });
+
+      const primary = status.members.find(member => member.stateStr === 'PRIMARY');
+      const secondary = status.members.find(member => member.stateStr === 'SECONDARY');
+
+      if (primary && secondary) {
+        break;
+      }
+    } while (Date.now() - startTime < timeout);
+
+    console.log('✅ Replica set is stable');
+  } catch (err) {
+    console.error('❌ Failed to initiate replica set:', err);
+  }
+}, 30000);
+
+afterAll(async () => {
+  await connection.close();
+});

migrations/20241111000000-add-payload-title-index-for-events.js

@@ -0,0 +1,55 @@
+/**
+ * This migration creates indexes for all collections on payload.title field
+ */
+
+/**
+ * Index name for payload.title field
+ */
+const payloadTitleIndexName = 'payloadTitle';
+
+module.exports = {
+  async up(db) {
+    const collections = await db.listCollections({}, {
+      authorizedCollections: true,
+      nameOnly: true,
+    }).toArray();
+
+    const targetCollections = [];
+
+    collections.forEach((collection) => {
+      if (/events/.test(collection.name)) {
+        targetCollections.push(collection.name);
+      }
+    });
+
+    for (const collectionName of targetCollections) {
+      const hasIndexAlready = await db.collection(collectionName).indexExists(payloadTitleIndexName);
+
+      if (!hasIndexAlready) {
+        await db.collection(collectionName).createIndex({
+          'payload.title': 1,
+        }, {
+          name: payloadTitleIndexName,
+        });
+      }
+    }
+  },
+  async down(db) {
+    const collections = await db.listCollections({}, {
+      authorizedCollections: true,
+      nameOnly: true,
+    }).toArray();
+
+    const targetCollections = [];
+
+    collections.forEach((collection) => {
+      if (/events/.test(collection.name)) {
+        targetCollections.push(collection.name);
+      }
+    });
+
+    for (const collectionName of targetCollections) {
+      await db.collection(collectionName).dropIndex(payloadTitleIndexName);
+    }
+  },
+};

package.json

@@ -48,7 +48,7 @@
     },
     "dependencies": {
         "@hawk.so/nodejs": "^3.1.1",
-        "@hawk.so/types": "^0.1.26",
+        "@hawk.so/types": "^0.1.29",
         "@types/amqplib": "^0.8.2",
         "@types/jest": "^29.2.3",
         "@types/mongodb": "^3.5.15",

workers/archiver/tests/index.test.ts

@@ -18,6 +18,7 @@ process.env.MAX_DAYS_NUMBER = '30';
 
 const mockedProject: ProjectDBScheme = {
   notifications: [],
+  eventGroupingPatterns: [],
   token: '5342',
   integrationId: 'eyJpbnRlZ3JhdGlvbklkIjoiMzg3NGNkOWMtZjJiYS00ZDVkLTk5ZmQtM2UzZjYzMDcxYmJhIiwic2VjcmV0IjoiMGZhM2JkM2EtYmMyZC00YWRiLThlMWMtNjg2OGY0MzM1YjRiIn0=',
   uidAdded: new ObjectId('5e4ff518628a6c714515f4db'),
@@ -53,7 +54,7 @@ describe('Archiver worker', () => {
 
   beforeEach(async () => {
     await db.collection('releases').deleteMany({});
-  })
+  });
 
   test('Should correctly remove old events', async () => {
     /**
@@ -129,7 +130,7 @@ describe('Archiver worker', () => {
     /**
      * Insert one release with object id based on current time, it should not be removed
      */
-    await db.collection('releases').insert(releasesToStay)
+    await db.collection('releases').insert(releasesToStay);
 
     const worker = new ArchiverWorker();
 
@@ -173,9 +174,9 @@ describe('Archiver worker', () => {
     expect(newReleasesCollection).toEqual([
       mockedReleases[mockedReleasesLength - 2],
       mockedReleases[mockedReleasesLength - 1],
-    ])
+    ]);
     await worker.finish();
-  })
+  });
 
   afterAll(async () => {
     await db.dropCollection('releases');