diff --git a/opencti-platform/opencti-graphql/config/default.json b/opencti-platform/opencti-graphql/config/default.json index afa19e1af384..0387ad29ae6c 100644 --- a/opencti-platform/opencti-graphql/config/default.json +++ b/opencti-platform/opencti-graphql/config/default.json @@ -288,7 +288,9 @@ "hub_registration_manager": { "enabled": true, "lock_key": "hub_registration_manager_lock", - "interval": 3600000 + "interval": 3600000, + "news_feed_cleanup_interval_value": 180, + "news_feed_cleanup_interval_unit": "days" }, "indicator_decay_manager": { "enabled": true, diff --git a/opencti-platform/opencti-graphql/src/manager/hubRegistrationManager.ts b/opencti-platform/opencti-graphql/src/manager/hubRegistrationManager.ts index 8e89057a6c37..32c6903f5a16 100644 --- a/opencti-platform/opencti-graphql/src/manager/hubRegistrationManager.ts +++ b/opencti-platform/opencti-graphql/src/manager/hubRegistrationManager.ts @@ -1,12 +1,34 @@ import { type ManagerDefinition, registerManager } from './managerModule'; -import conf, { booleanConf } from '../config/conf'; +import conf, { booleanConf, logApp } from '../config/conf'; import { executionContext, HUB_REGISTRATION_MANAGER_USER } from '../utils/access'; import { checkXTMHubConnectivity, loadAndSaveLatestNewsFeed } from '../domain/xtm-hub'; import { XtmHubRegistrationStatus } from '../generated/graphql'; +import { cleanOldNewsFeedItems } from '../modules/xtm/hub/news-feed/news-feed-domain'; +import moment from 'moment'; const HUB_REGISTRATION_MANAGER_ENABLED = booleanConf('hub_registration_manager:enabled', true); const HUB_REGISTRATION_MANAGER_KEY = conf.get('hub_registration_manager:lock_key') || 'hub_registration_manager_lock'; const SCHEDULE_TIME = conf.get('hub_registration_manager:interval') || 60 * 60 * 1000; // 1 hour +const NEWS_FEED_CLEANUP_INTERVAL_VALUE = conf.get('hub_registration_manager:news_feed_cleanup_interval_value') || 180; +const NEWS_FEED_CLEANUP_INTERVAL_UNIT = conf.get('hub_registration_manager:news_feed_cleanup_interval_unit') || 'days'; + +const VALID_CLEANUP_UNITS = ['seconds', 'minutes', 'hours', 'days', 'weeks', 'months', 'years'] as const; +type ValidCleanupUnit = typeof VALID_CLEANUP_UNITS[number]; + +if ( + typeof NEWS_FEED_CLEANUP_INTERVAL_VALUE !== 'number' + || !Number.isFinite(NEWS_FEED_CLEANUP_INTERVAL_VALUE) + || NEWS_FEED_CLEANUP_INTERVAL_VALUE <= 0 +) { + throw new Error( + `[XTMH] Invalid news_feed_cleanup_interval_value: expected a positive number, got "${NEWS_FEED_CLEANUP_INTERVAL_VALUE}"`, + ); +} +if (!VALID_CLEANUP_UNITS.includes(NEWS_FEED_CLEANUP_INTERVAL_UNIT as ValidCleanupUnit)) { + throw new Error( + `[XTMH] Invalid news_feed_cleanup_interval_unit: "${NEWS_FEED_CLEANUP_INTERVAL_UNIT}". Expected one of: ${VALID_CLEANUP_UNITS.join(', ')}`, + ); +} /** * If platform is registered, calls XTM Hub backend to check if the registration data is still valid @@ -18,6 +40,26 @@ export const hubRegistrationManager = async () => { if (status === XtmHubRegistrationStatus.Registered) { await loadAndSaveLatestNewsFeed(context, HUB_REGISTRATION_MANAGER_USER); } + try { + const cutoffDate = moment() + .subtract(NEWS_FEED_CLEANUP_INTERVAL_VALUE, NEWS_FEED_CLEANUP_INTERVAL_UNIT as ValidCleanupUnit) + .toDate(); + const deletedCount = await cleanOldNewsFeedItems( + context, + HUB_REGISTRATION_MANAGER_USER, + cutoffDate, + ); + if (deletedCount > 0) { + logApp.info('[XTMH] Cleaned expired news feed items', { + deletedCount, + cutoffDate: cutoffDate.toISOString(), + intervalValue: NEWS_FEED_CLEANUP_INTERVAL_VALUE, + intervalUnit: NEWS_FEED_CLEANUP_INTERVAL_UNIT, + }); + } + } catch (err) { + logApp.error('[XTMH] Failed to clean expired news feed items', { cause: err }); + } }; const HUB_REGISTRATION_MANAGER_DEFINITION: ManagerDefinition = { @@ -37,5 +79,4 @@ const HUB_REGISTRATION_MANAGER_DEFINITION: ManagerDefinition = { return this.enabledByConfig; }, }; - registerManager(HUB_REGISTRATION_MANAGER_DEFINITION); diff --git a/opencti-platform/opencti-graphql/src/modules/xtm/hub/news-feed/news-feed-domain.ts b/opencti-platform/opencti-graphql/src/modules/xtm/hub/news-feed/news-feed-domain.ts index 14649a952e77..7a3cb6ed44a8 100644 --- a/opencti-platform/opencti-graphql/src/modules/xtm/hub/news-feed/news-feed-domain.ts +++ b/opencti-platform/opencti-graphql/src/modules/xtm/hub/news-feed/news-feed-domain.ts @@ -4,9 +4,13 @@ import type { NewsFeedAddInput } from './news-feed-types'; import { createInternalObject } from '../../../../domain/internalObject'; import { addFilter } from '../../../../utils/filtering/filtering-utils'; import { pageEntitiesConnection } from '../../../../database/middleware-loader'; -import { elCount } from '../../../../database/engine'; +import { elCount, elPaginate } from '../../../../database/engine'; import { READ_INDEX_INTERNAL_OBJECTS } from '../../../../database/utils'; -import type { QueryMyNewsFeedsArgs } from '../../../../generated/graphql'; +import { FilterMode, FilterOperator, type QueryMyNewsFeedsArgs } from '../../../../generated/graphql'; +import { deleteElementById } from '../../../../database/middleware'; +import { ALREADY_DELETED_ERROR } from '../../../../config/errors'; +import { logApp } from '../../../../config/conf'; +import { promiseMap } from '../../../../utils/promiseUtils'; export const addNewsFeed = async (context: AuthContext, user: AuthUser, input: NewsFeedAddInput) => { const newsFeedToCreate = { @@ -35,3 +39,67 @@ export const myUnreadNewsFeedsCount = (context: AuthContext, user: AuthUser) => const queryArgs = { filters: queryFilters, types: [ENTITY_TYPE_NEWS_FEED_ITEM] }; return elCount(context, user, READ_INDEX_INTERNAL_OBJECTS, queryArgs); }; + +const NEWS_FEED_CLEANUP_BATCH_SIZE = 1500; +const NEWS_FEED_CLEANUP_CONCURRENCY = 5; + +export const cleanOldNewsFeedItems = async ( + context: AuthContext, + user: AuthUser, + cutoffDate: Date, +): Promise => { + const filters = { + mode: FilterMode.And, + filters: [ + { + key: ['creation_date'], + values: [cutoffDate.toISOString()], + operator: FilterOperator.Lt, + }, + ], + filterGroups: [], + }; + + let totalDeleted = 0; + + while (true) { + const result = await elPaginate(context, user, READ_INDEX_INTERNAL_OBJECTS, { + filters, + types: [ENTITY_TYPE_NEWS_FEED_ITEM], + first: NEWS_FEED_CLEANUP_BATCH_SIZE, + }) as any; + + if (!result.edges || result.edges.length === 0) { + break; + } + + await promiseMap( + result.edges, + async (edge: any) => { + try { + await deleteElementById( + context, + user, + edge.node.internal_id, + ENTITY_TYPE_NEWS_FEED_ITEM, + ); + totalDeleted += 1; + } catch (err: any) { + if (err?.extensions?.code !== ALREADY_DELETED_ERROR) { + logApp.error('[XTMH] Failed to delete news feed item during cleanup', { + cause: err, + id: edge.node.internal_id, + }); + } + } + }, + NEWS_FEED_CLEANUP_CONCURRENCY, + ); + + if (result.edges.length < NEWS_FEED_CLEANUP_BATCH_SIZE) { + break; + } + } + + return totalDeleted; +}; diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/hubRegistrationManager-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/hubRegistrationManager-test.ts index e9345288e9b1..83c8dc515f79 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/hubRegistrationManager-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/hubRegistrationManager-test.ts @@ -21,9 +21,15 @@ vi.mock('../../../src/manager/managerModule', () => ({ registerManager: vi.fn(), })); +vi.mock('../../../src/modules/xtm/hub/news-feed/news-feed-domain', () => ({ + cleanOldNewsFeedItems: vi.fn(), +})); + import { hubRegistrationManager } from '../../../src/manager/hubRegistrationManager'; import { checkXTMHubConnectivity, loadAndSaveLatestNewsFeed } from '../../../src/domain/xtm-hub'; +import { cleanOldNewsFeedItems } from '../../../src/modules/xtm/hub/news-feed/news-feed-domain'; +const mockCleanOldNewsFeedItems = vi.mocked(cleanOldNewsFeedItems); const mockCheckXTMHubConnectivity = vi.mocked(checkXTMHubConnectivity); const mockLoadAndSaveLatestNewsFeed = vi.mocked(loadAndSaveLatestNewsFeed); @@ -80,4 +86,13 @@ describe('hubRegistrationManager', () => { await expect(hubRegistrationManager()).rejects.toThrow('Failed to load news feed'); }); + + it('should call cleanOldNewsFeedItems on every execution', async () => { + mockCheckXTMHubConnectivity.mockResolvedValue({ status: XtmHubRegistrationStatus.Unregistered }); + mockCleanOldNewsFeedItems.mockResolvedValue(0); + + await hubRegistrationManager(); + + expect(mockCleanOldNewsFeedItems).toHaveBeenCalledOnce(); + }); }); diff --git a/opencti-platform/opencti-graphql/tests/01-unit/domain/newsFeed-test.ts b/opencti-platform/opencti-graphql/tests/01-unit/domain/newsFeed-test.ts index c1f827050898..1426113aa207 100644 --- a/opencti-platform/opencti-graphql/tests/01-unit/domain/newsFeed-test.ts +++ b/opencti-platform/opencti-graphql/tests/01-unit/domain/newsFeed-test.ts @@ -1,13 +1,16 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { addNewsFeed, myNewsFeedsFind, myUnreadNewsFeedsCount } from '../../../src/modules/xtm/hub/news-feed/news-feed-domain'; +import { addNewsFeed, cleanOldNewsFeedItems, myNewsFeedsFind, myUnreadNewsFeedsCount } from '../../../src/modules/xtm/hub/news-feed/news-feed-domain'; import { NewsFeedItemType } from '../../../src/modules/xtm/hub/news-feed/news-feed-types'; import type { NewsFeedAddInput } from '../../../src/modules/xtm/hub/news-feed/news-feed-types'; +import { ALREADY_DELETED_ERROR } from '../../../src/config/errors'; const mockCreateInternalObject = vi.fn(); const mockPageEntitiesConnection = vi.fn(); const mockElCount = vi.fn(); const mockFullEntitiesList = vi.fn(); const mockPatchAttribute = vi.fn(); +const mockElPaginate = vi.fn(); +const mockDeleteElementById = vi.fn(); vi.mock('../../../src/domain/internalObject', () => ({ createInternalObject: (...args: unknown[]) => mockCreateInternalObject(...args), @@ -20,10 +23,12 @@ vi.mock('../../../src/database/middleware-loader', () => ({ vi.mock('../../../src/database/engine', () => ({ elCount: (...args: unknown[]) => mockElCount(...args), + elPaginate: (...args: unknown[]) => mockElPaginate(...args), })); vi.mock('../../../src/database/middleware', () => ({ patchAttribute: (...args: unknown[]) => mockPatchAttribute(...args), + deleteElementById: (...args: unknown[]) => mockDeleteElementById(...args), })); vi.mock('../../../src/utils/filtering/filtering-utils', () => ({ @@ -196,4 +201,90 @@ describe('News feed', () => { expect(result).toBe(3); }); }); + + describe('cleanOldNewsFeedItems', () => { + const cutoffDate = new Date('2026-05-01T00:00:00.000Z'); + + beforeEach(() => { + mockElPaginate.mockReset(); + mockDeleteElementById.mockReset(); + }); + + it('should call elPaginate with creation_date < cutoff filter', async () => { + mockElPaginate.mockResolvedValue({ edges: [] }); + + await cleanOldNewsFeedItems(mockContext, mockUser, cutoffDate); + + expect(mockElPaginate).toHaveBeenCalledWith( + mockContext, + mockUser, + expect.any(String), + expect.objectContaining({ + filters: expect.objectContaining({ + filters: expect.arrayContaining([ + expect.objectContaining({ + key: ['creation_date'], + values: [cutoffDate.toISOString()], + operator: 'lt', + }), + ]), + }), + types: ['NewsFeedItem'], + }), + ); + }); + + it('should delete each returned item and return the count', async () => { + mockElPaginate + .mockResolvedValueOnce({ + edges: [ + { node: { internal_id: 'item-1' } }, + { node: { internal_id: 'item-2' } }, + { node: { internal_id: 'item-3' } }, + ], + }); + mockDeleteElementById.mockResolvedValue(undefined); + + const result = await cleanOldNewsFeedItems(mockContext, mockUser, cutoffDate); + + expect(mockDeleteElementById).toHaveBeenCalledTimes(3); + expect(mockDeleteElementById).toHaveBeenCalledWith( + mockContext, + mockUser, + 'item-1', + 'NewsFeedItem', + ); + expect(result).toBe(3); + }); + + it('should return 0 and not call delete when no items match', async () => { + mockElPaginate.mockResolvedValue({ edges: [] }); + + const result = await cleanOldNewsFeedItems(mockContext, mockUser, cutoffDate); + + expect(result).toBe(0); + expect(mockDeleteElementById).not.toHaveBeenCalled(); + }); + + it('should swallow ALREADY_DELETED_ERROR without breaking the loop', async () => { + mockElPaginate.mockResolvedValueOnce({ + edges: [ + { node: { internal_id: 'item-1' } }, + { node: { internal_id: 'item-2' } }, + ], + }); + const alreadyDeletedError = Object.assign(new Error('Already deleted'), { + extensions: { code: ALREADY_DELETED_ERROR }, + }); + mockDeleteElementById + .mockRejectedValueOnce(alreadyDeletedError) + .mockResolvedValueOnce(undefined); + + const result = await cleanOldNewsFeedItems(mockContext, mockUser, cutoffDate); + + // Le 1er fail silencieusement (n'incrémente pas), le 2e réussit + expect(result).toBe(1); + expect(mockDeleteElementById).toHaveBeenCalledTimes(2); + }); + }); });