From 9cec701cb4097abd9fdb5b994e718d7a24fa1850 Mon Sep 17 00:00:00 2001 From: Vyacheslav Chernyshev <81693471+slaveeks@users.noreply.github.com> Date: Thu, 11 Sep 2025 21:42:33 +0300 Subject: [PATCH 1/4] chore: added try-catch for sending report (#457) * chore: added try-catch for sending report * added res log --- workers/archiver/src/index.ts | 37 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/workers/archiver/src/index.ts b/workers/archiver/src/index.ts index 39d07e41..d45c863f 100644 --- a/workers/archiver/src/index.ts +++ b/workers/archiver/src/index.ts @@ -103,13 +103,18 @@ export default class ArchiverWorker extends Worker { const finishDate = new Date(); const dbSizeOnFinish = (await this.eventsDbConnection.stats()).dataSize; - await this.sendReport({ - dbSizeOnFinish, - dbSizeOnStart, - startDate, - projectsData, - finishDate, - }); + try { + await this.sendReport({ + dbSizeOnFinish, + dbSizeOnStart, + startDate, + projectsData, + finishDate, + }); + } catch (error) { + this.logger.error('Error sending report:', error); + } + this.logger.info(`Finish archiving at ${finishDate}.`); this.logger.info(`Database size on start: ${prettysize(dbSizeOnStart)}, on finish: ${prettysize(dbSizeOnFinish)}, delta: ${prettysize(dbSizeOnStart - dbSizeOnFinish)}`); } @@ -148,11 +153,11 @@ export default class ArchiverWorker extends Worker { await this.projectCollection.updateOne({ _id: project._id, }, - { - $inc: { - archivedEventsCount: deletedCount, - }, - }); + { + $inc: { + archivedEventsCount: deletedCount, + }, + }); } /** @@ -335,11 +340,17 @@ export default class ArchiverWorker extends Worker { report += `\n\n${totalArchivedEventsCount} events and ${totalRemovedReleasesCount} releases archived in ${archivingTimeInMinutes.toFixed(DIGITS_AFTER_POINT)} min`; report += `\nDatabase size changed from ${prettysize(reportData.dbSizeOnStart)} to ${prettysize(reportData.dbSizeOnFinish)} (–${prettysize(reportData.dbSizeOnStart - reportData.dbSizeOnFinish)})`; - await axios({ + const response = await axios({ method: 'post', url: process.env.REPORT_NOTIFY_URL, data: 'message=' + report + '&parse_mode=HTML', }); + + this.logger.info('Report notification response:', { + status: response.status, + statusText: response.statusText, + data: response.data + }); } /** From 955688376e7c76643ea2ff06c3d4ba7d58d432fb Mon Sep 17 00:00:00 2001 From: Dobrunia Kostrigin <48620984+Dobrunia@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:18:21 +0300 Subject: [PATCH 2/4] chore(license): adopt BSL 1.1 (#460) * chore: update license to BUSL-1.1 in package.json and README.md * chore: update licensed work reference in LICENSE file --- LICENSE | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 7 ++++ package.json | 2 +- 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..0f13dca4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,96 @@ +Business Source License 1.1 + +Parameters + +Licensor: CodeX (Hawk) +Licensed Work: hawk.workers (https://github.com/codex-team/hawk.workers) + The Licensed Work is © 2025 CodeX +Additional Use Grant: Self-hosted use for own/internal needs and research/evaluation is permitted. + Using this code to provide a competing error-tracking SaaS or commercial + hosted service without the Licensor’s prior permission is prohibited. + +Change Date: 2030-01-01 + +Change License: AGPL-3.0 + +------------------------------------------------------------------------------- + +License text copyright © 2024 MariaDB plc, All Rights Reserved. +“Business Source License” is a trademark of MariaDB plc. + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). TO THE EXTENT PERMITTED BY +APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN “AS IS” BASIS. LICENSOR +HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS OR IMPLIED, INCLUDING +(WITHOUT LIMITATION) WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE, NON-INFRINGEMENT, AND TITLE. MariaDB hereby grants you permission to +use this License’s text to license your works, and to refer to it using the +trademark “Business Source License”, as long as you comply with the Covenants +of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date not later than the fourth anniversary of the first + publicly available distribution of a specific version of the Licensed Work + under this License. + +4. Not to modify this License in any other way. + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +For more information on the use of the Business Source License for MariaDB +products, please visit the MariaDB Business Source License FAQ. +For more information on the use of the Business Source License generally, +please visit the Adopting and Developing Business Source License FAQ. diff --git a/README.md b/README.md index c6fd3846..371a196e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ # Hawk Workers +![License](https://img.shields.io/badge/license-BSL--1.1-orange) + Workers are services for processing hawk's background tasks @@ -219,3 +221,8 @@ yarn migrate Refactor mongo-migrate commands to have an opportunity to create or rollback [More details](https://www.npmjs.com/package/migrate-mongo) + +## License + +Source code is available under **Business Source License 1.1 (BSL 1.1)**. +See [`LICENSE`](./LICENSE) for terms, including: diff --git a/package.json b/package.json index d27a60ab..9f930d5c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "description": "Hawk workers", "repository": "git@github.com:codex-team/hawk.workers.git", - "license": "UNLICENSED", + "license": "BUSL-1.1", "workspaces": [ "workers/*" ], From 1ec06628a041329ac3a09aa24816ea5bf2f9efae Mon Sep 17 00:00:00 2001 From: e11sy <130844513+e11sy@users.noreply.github.com> Date: Tue, 23 Sep 2025 23:11:15 +0300 Subject: [PATCH 3/4] imp(): get rid of duplicated releases (#462) * imp(): get rid of duplicated releases * chore(): eslint fix --- ...7000000-create-release-project-id-index.js | 105 ++++++++++++ workers/release/src/index.ts | 158 +++++++++--------- 2 files changed, 187 insertions(+), 76 deletions(-) create mode 100644 migrations/20250917000000-create-release-project-id-index.js diff --git a/migrations/20250917000000-create-release-project-id-index.js b/migrations/20250917000000-create-release-project-id-index.js new file mode 100644 index 00000000..9fee1934 --- /dev/null +++ b/migrations/20250917000000-create-release-project-id-index.js @@ -0,0 +1,105 @@ +const indexName = 'projectId_release_unique_idx'; +const collectionName = 'releases'; + +module.exports = { + async up(db) { + const pairs = await db.collection(collectionName).aggregate([ + { + $group: { + _id: { projectId: '$projectId', release: '$release' }, + count: { $sum: 1 }, + }, + }, + { $project: { _id: 0, projectId: '$_id.projectId', release: '$_id.release', count: 1 } }, + ]).toArray(); + + console.log(`Found ${pairs.length} unique (projectId, release) pairs to process.`); + + let processed = 0; + + for (const { projectId, release, count } of pairs) { + processed += 1; + console.log(`[${processed}/${pairs.length}] Processing projectId=${projectId}, release=${release} (docs: ${count})`); + + try { + const docs = await db.collection(collectionName) + .find({ projectId, release }, { projection: { files: 1, commits: 1 } }) + .toArray(); + + const filesByName = new Map(); + const commitsByHash = new Map(); + + for (const doc of docs) { + if (Array.isArray(doc.files)) { + for (const file of doc.files) { + /** + * Keep first occurrence if duplicates conflict + */ + if (file && typeof file === 'object' && file.mapFileName && !filesByName.has(file.mapFileName)) { + filesByName.set(file.mapFileName, file); + } + } + } + if (Array.isArray(doc.commits)) { + for (const commit of doc.commits) { + if (commit && typeof commit === 'object' && commit.hash && !commitsByHash.has(commit.hash)) { + commitsByHash.set(commit.hash, commit); + } + } + } + } + + const mergedFiles = Array.from(filesByName.values()); + const mergedCommits = Array.from(commitsByHash.values()); + + /** + * Replace all docs for this pair with a single consolidated doc + */ + const ops = [ + { deleteMany: { filter: { projectId, release } } }, + { insertOne: { document: { projectId, release, files: mergedFiles, commits: mergedCommits } } }, + ]; + + await db.collection(collectionName).bulkWrite(ops, { ordered: true }); + console.log(`Consolidated projectId=${projectId}, release=${release}: files=${mergedFiles.length}, commits=${mergedCommits.length}`); + } catch (err) { + console.error(`Error consolidating projectId=${projectId}, release=${release}:`, err); + } + } + + /** + * Create the unique compound index + */ + try { + const hasIndex = await db.collection(collectionName).indexExists(indexName); + if (!hasIndex) { + await db.collection(collectionName).createIndex( + { projectId: 1, release: 1 }, + { name: indexName, unique: true, background: true } + ); + console.log(`Index ${indexName} created on ${collectionName} (projectId, release unique).`); + } else { + console.log(`Index ${indexName} already exists on ${collectionName}.`); + } + } catch (err) { + console.error(`Error creating index ${indexName} on ${collectionName}:`, err); + } + + }, + + async down(db) { + console.log(`Dropping index ${indexName} from ${collectionName}...`); + try { + const hasIndex = await db.collection(collectionName).indexExists(indexName); + if (hasIndex) { + await db.collection(collectionName).dropIndex(indexName); + console.log(`Index ${indexName} dropped from ${collectionName}.`); + } else { + console.log(`Index ${indexName} does not exist on ${collectionName}, skipping drop.`); + } + } catch (err) { + console.error(`Error dropping index ${indexName} from ${collectionName}:`, err); + } + console.log('Down migration completed (data changes are not reverted).'); + }, +}; diff --git a/workers/release/src/index.ts b/workers/release/src/index.ts index 3359dda7..4a46b63a 100644 --- a/workers/release/src/index.ts +++ b/workers/release/src/index.ts @@ -9,8 +9,15 @@ import { Worker } from '../../../lib/worker'; import { DatabaseReadWriteError, NonCriticalError } from '../../../lib/workerErrors'; import * as pkg from '../package.json'; import { ReleaseWorkerTask, ReleaseWorkerAddReleasePayload, CommitDataUnparsed } from '../types'; -import { Collection, MongoClient } from 'mongodb'; +import { Collection, MongoClient, MongoError } from 'mongodb'; import { SourceMapDataExtended, SourceMapFileChunk, CommitData, SourcemapCollectedData, ReleaseDBScheme } from '@hawk.so/types'; + +/** + * Error code of MongoDB key duplication error + */ +/* eslint-disable @typescript-eslint/no-magic-numbers */ +const DB_DUPLICATE_KEY_ERROR = '11000'; + /** * Worker to save releases */ @@ -142,105 +149,104 @@ export default class ReleaseWorker extends Worker { * @param payload - source map data */ private async saveSourceMap(projectId: string, payload: ReleaseWorkerAddReleasePayload): Promise { + const files: SourceMapDataExtended[] = this.extendReleaseInfo(payload.files); + /** - * Start transaction to avoid race condition + * Use same transaction for read and related write operations */ - const session = await this.client.startSession(); - - try { - const files: SourceMapDataExtended[] = this.extendReleaseInfo(payload.files); + const existedRelease = await this.releasesCollection.findOne({ + projectId: projectId, + release: payload.release, + }); + /** + * Iterate all maps of the new release and save only new + */ + const savedFiles = await Promise.all(files.map(async (map: SourceMapDataExtended) => { /** - * Use same transaction for read and related write operations + * Skip already saved maps */ - await session.withTransaction(async () => { - const existedRelease = await this.releasesCollection.findOne({ - projectId: projectId, - release: payload.release, - }, { session }); - - /** - * Iterate all maps of the new release and save only new - */ - const savedFiles = await Promise.all(files.map(async (map: SourceMapDataExtended) => { - /** - * Skip already saved maps - */ - - const alreadySaved = existedRelease && existedRelease.files && existedRelease.files.find((savedFile) => { - return savedFile.mapFileName === map.mapFileName; - }); + const alreadySaved = existedRelease && existedRelease.files && !!existedRelease.files.find((savedFile) => { + return savedFile.mapFileName === map.mapFileName; + }); - if (alreadySaved) { - return; - } + if (alreadySaved) { + return; + } - try { - const fileInfo = await this.saveFile(map); - - /** - * Save id of saved file instead - */ - return { - ...map, - _id: fileInfo._id, - }; - } catch (error) { - this.logger.error(`Map ${map.mapFileName} was not saved: ${error}`); - } - })); + try { + const fileInfo = await this.saveFile(map); /** - * Filter undefined files and then prepare files that would be saved to releases table - * we do not need their content since it would be stored in gridFS + * Save id of saved file instead */ - const savedFilesWithoutContent: Omit[] = savedFiles.filter(file => { - return file !== undefined; - }).map(({ content, ...rest }) => { - return rest; - }); + return { + ...map, + _id: fileInfo._id, + }; + } catch (error) { + this.logger.error(`Map ${map.mapFileName} was not saved: ${error}`); + } + })); - /** - * Nothing to save: maps was previously saved - */ - if (savedFilesWithoutContent.length === 0) { - return; - } + /** + * Filter undefined files and then prepare files that would be saved to releases table + * we do not need their content since it would be stored in gridFS + */ + const savedFilesWithoutContent: Omit[] = savedFiles.filter(file => { + return file !== undefined; + }).map(({ content, ...rest }) => { + return rest; + }); - /** - * - insert new record with saved maps - * or - * - update previous record with adding new saved maps - */ - if (!existedRelease) { - this.logger.info('inserted new release'); + /** + * Nothing to save: maps was previously saved + */ + if (savedFilesWithoutContent.length === 0) { + return; + } + + try { + /** + * - insert new record with saved maps + * or + * - update previous record with adding new saved maps + */ + if (!existedRelease) { + this.logger.info('trying insert new release'); + + try { await this.releasesCollection.insertOne({ projectId: projectId, release: payload.release, files: savedFilesWithoutContent, - } as ReleaseDBScheme, { session }); + } as ReleaseDBScheme); + this.logger.info('inserted new release'); + } catch (err) { + if ((err as MongoError).code.toString() === DB_DUPLICATE_KEY_ERROR) { + this.logger.warn(`Duplicate key on insert, retrying update after small delay`); + /* eslint-disable @typescript-eslint/no-magic-numbers */ + await new Promise(resolve => setTimeout(resolve, 200)); + } else { + throw err; + } } + } - await this.releasesCollection.findOneAndUpdate({ - projectId: projectId, - release: payload.release, - }, { - $push: { - files: { - $each: savedFilesWithoutContent, - }, + await this.releasesCollection.findOneAndUpdate({ + projectId: projectId, + release: payload.release, + }, { + $push: { + files: { + $each: savedFilesWithoutContent, }, - }, { session }); + }, }); } catch (error) { this.logger.error(`Can't extract release info:\n${JSON.stringify(error)}`); throw new NonCriticalError('Can\'t parse source-map file'); - } finally { - /** - * End transaction - */ - await session.endSession(); } } From b3d1b0ce788b0f6121cd41bb465760af020f31c9 Mon Sep 17 00:00:00 2001 From: Dobrunia Kostrigin <48620984+Dobrunia@users.noreply.github.com> Date: Wed, 1 Oct 2025 22:22:17 +0300 Subject: [PATCH 4/4] fix(grouper): skip notifications for muted errors (#459) * feat(grouper): add event ignored check before notifying worker * Implemented a new method to check if an event is marked as ignored. * Updated the notifier task addition to only proceed if the event is not ignored. * Removed the isEventIgnored method as its functionality is now integrated directly into the task addition logic. * Removed unnecessary line break * update @hawk.so/types dependency to version 0.1.35 * add tests for event marks handling in NotifierWorker * fix --- package.json | 2 +- workers/grouper/src/index.ts | 30 ++++++++---- workers/grouper/tests/index.test.ts | 72 +++++++++++++++++++++++++++++ yarn.lock | 8 ++-- 4 files changed, 97 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 9f930d5c..9f094dc2 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ }, "dependencies": { "@hawk.so/nodejs": "^3.1.1", - "@hawk.so/types": "^0.1.32", + "@hawk.so/types": "^0.1.35", "@types/amqplib": "^0.8.2", "@types/jest": "^29.2.3", "@types/mongodb": "^3.5.15", diff --git a/workers/grouper/src/index.ts b/workers/grouper/src/index.ts index 2a89aed8..f7621149 100644 --- a/workers/grouper/src/index.ts +++ b/workers/grouper/src/index.ts @@ -223,20 +223,30 @@ export default class GrouperWorker extends Worker { /** * Store events counter by days */ - await this.saveDailyEvents(task.projectId, uniqueEventHash, task.timestamp, repetitionId, incrementDailyAffectedUsers); + await this.saveDailyEvents( + task.projectId, + uniqueEventHash, + task.timestamp, + repetitionId, + incrementDailyAffectedUsers + ); /** - * Add task for NotifierWorker + * Add task for NotifierWorker only if event is not ignored */ if (process.env.IS_NOTIFIER_WORKER_ENABLED) { - await this.addTask(WorkerNames.NOTIFIER, { - projectId: task.projectId, - event: { - title: task.payload.title, - groupHash: uniqueEventHash, - isNew: isFirstOccurrence, - }, - }); + const isIgnored = isFirstOccurrence ? false : !!existedEvent?.marks?.ignored; + + if (!isIgnored) { + await this.addTask(WorkerNames.NOTIFIER, { + projectId: task.projectId, + event: { + title: task.payload.title, + groupHash: uniqueEventHash, + isNew: isFirstOccurrence, + }, + }); + } } } diff --git a/workers/grouper/tests/index.test.ts b/workers/grouper/tests/index.test.ts index cd410fc2..6d1c9adc 100644 --- a/workers/grouper/tests/index.test.ts +++ b/workers/grouper/tests/index.test.ts @@ -589,6 +589,78 @@ describe('GrouperWorker', () => { }); }); + describe('Event marks handling', () => { + describe('Ignored events', () => { + it('should not add task for NotifierWorker when event is marked as ignored', async () => { + const mockAddTask = jest + .spyOn(worker as any, 'addTask') + .mockImplementation(() => Promise.resolve()); + + // Create an event first + const firstTask = generateTask({ title: 'Test ignored event' }); + await worker.handle(firstTask); + + // Mark the event as ignored by updating it in database + const eventHash = await (worker as any).getUniqueEventHash(firstTask); + await eventsCollection.updateOne( + { groupHash: eventHash }, + { $set: { marks: { ignored: true } } } + ); + + // Handle the same event again (repetition) + const secondTask = generateTask({ title: 'Test ignored event' }); + await worker.handle(secondTask); + + // Verify that addTask was called only once (for the first occurrence) + expect(mockAddTask).toHaveBeenCalledTimes(1); + + mockAddTask.mockRestore(); + }); + + it('should add task for NotifierWorker when event is not marked as ignored', async () => { + const mockAddTask = jest + .spyOn(worker as any, 'addTask') + .mockImplementation(() => Promise.resolve()); + + // Create an event first + const firstTask = generateTask({ title: 'Test non-ignored event' }); + await worker.handle(firstTask); + + // Handle the same event again (repetition) - without marking as ignored + const secondTask = generateTask({ title: 'Test non-ignored event' }); + await worker.handle(secondTask); + + // Verify that addTask was called twice (for both occurrences) + expect(mockAddTask).toHaveBeenCalledTimes(2); + + mockAddTask.mockRestore(); + }); + + it('should add task for NotifierWorker for first occurrence even if marks field is undefined', async () => { + const mockAddTask = jest + .spyOn(worker as any, 'addTask') + .mockImplementation(() => Promise.resolve()); + + // Create a new event (first occurrence) + const task = generateTask({ title: 'Test new event without marks' }); + await worker.handle(task); + + // Verify that addTask was called for the first occurrence + expect(mockAddTask).toHaveBeenCalledTimes(1); + expect(mockAddTask).toHaveBeenCalledWith('notifier', { + projectId: task.projectId, + event: { + title: task.payload.title, + groupHash: expect.any(String), + isNew: true, + }, + }); + + mockAddTask.mockRestore(); + }); + }); + }); + afterAll(async () => { await redisClient.quit(); await worker.finish(); diff --git a/yarn.lock b/yarn.lock index 2180bae4..13207d35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -428,10 +428,10 @@ dependencies: "@types/mongodb" "^3.5.34" -"@hawk.so/types@^0.1.32": - version "0.1.32" - resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.32.tgz#5eb662e91da922e1cbab7af0cf03bfa567ee98df" - integrity sha512-7gdx/fq31iLxmc0hZKs+wPYqtRT4rLvTi9p4am2My9SQ4t/l8if5QEfxh6G4WABKOmhiZLchivFA9Oj+Nn5EcQ== +"@hawk.so/types@^0.1.35": + version "0.1.35" + resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.35.tgz#6afd416dced1cc3282d721ca5621bf452b27aea1" + integrity sha512-uMTAeu6DlRlk+oputJBjTlrm1GzOkIwlMfGhpdOp3sRWe/YPGD6nMYlb9MZoVN6Yee7RIpYD7It+DPeUPAyIFw== dependencies: "@types/mongodb" "^3.5.34"