diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index dbafa020706f8..1b5bac10a65df 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -102,11 +102,21 @@ settings.watch('API_Enable_Rate_Limiter_Limit_Calls_Default', (value) => export const startRestAPI = () => { (WebApp.rawConnectHandlers as unknown as ReturnType).use( API.api + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\/v1\//), + api: API.v1, + settings, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) + .use(tracerSpanMiddleware) .use(remoteAddressMiddleware) .use(cors(settings)) .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi })) - .use(tracerSpanMiddleware) .use(API.v1.router) .use(API.default.router).router, ); diff --git a/apps/meteor/app/api/server/middlewares/metrics.spec.ts b/apps/meteor/app/api/server/middlewares/metrics.spec.ts index 6f53a489c6fb5..3fdc94cda826a 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.spec.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.spec.ts @@ -25,27 +25,42 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); - api.use(metricsMiddleware({ api: { version: 1 } as any, settings, summary: summary as any })).get( - '/test', - { - response: { - 200: ajv.compile({ - type: 'object', - properties: { - message: { type: 'string' }, + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + + api + .use( + metricsMiddleware({ + api: { version: 1 } as any, + settings, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) + .get( + '/test', + { + response: { + 200: ajv.compile({ + type: 'object', + properties: { + message: { type: 'string' }, + }, + }), + }, + }, + async () => { + return { + statusCode: 200, + body: { + message: 'Metrics test successful', }, - }), + }; }, - }, - async () => { - return { - statusCode: 200, - body: { - message: 'Metrics test successful', - }, - }; - }, - ); + ); app.use(api.router); const response = await request(app).get('/api/test').set('user-agent', 'test'); expect(response.statusCode).toBe(200); @@ -73,8 +88,22 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, summary: summary as any })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\//), + api: { version: 1 } as any, + settings, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) .get( '/test', { @@ -120,8 +149,22 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, summary: summary as any })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\//), + api: { version: 1 } as any, + settings, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) .get( '/method.call/:id', { diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index bedc2cc823a93..75774c495dbaf 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -1,5 +1,5 @@ import type { MiddlewareHandler } from 'hono'; -import type { Summary } from 'prom-client'; +import type { Gauge, Histogram, Summary } from 'prom-client'; import type { CachedSettings } from '../../../settings/server/CachedSettings'; import type { APIClass } from '../ApiClass'; @@ -9,28 +9,51 @@ export const metricsMiddleware = basePathRegex, api, settings, - summary, + endpointTimeSummary, + endpointTimeHistogram, + responseSizeHistogram, + activeRequestsGauge, }: { basePathRegex?: RegExp; api: APIClass; settings: CachedSettings; - summary: Summary; + endpointTimeSummary: Summary; + endpointTimeHistogram: Histogram; + responseSizeHistogram: Histogram; + activeRequestsGauge: Gauge; }): MiddlewareHandler => async (c, next) => { - const rocketchatRestApiEnd = summary.startTimer(); + const rocketchatRestApiEnd = endpointTimeSummary.startTimer(); + const rocketchatRestApiHistEnd = endpointTimeHistogram.startTimer(); + + const methodLabel = { method: c.req.method.toLowerCase() }; + activeRequestsGauge.inc(methodLabel); await next(); + activeRequestsGauge.dec(methodLabel); + const { method, path, routePath } = c.req; // get rid of the base path (i.e.: /api/v1/) const entrypoint = basePathRegex ? routePath.replace(basePathRegex, '') : routePath; - rocketchatRestApiEnd({ + const histogramLabels = { status: c.res.status, method: method.toLowerCase(), version: api.version, - ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), entrypoint: basePathRegex && entrypoint.startsWith('method.call') ? decodeURIComponent(path.replace(basePathRegex, '')) : entrypoint, + }; + + rocketchatRestApiEnd({ + ...histogramLabels, + ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), }); + + rocketchatRestApiHistEnd(histogramLabels); + + const contentLength = parseInt(c.res.headers.get('content-length') || '0', 10); + if (contentLength > 0) { + responseSizeHistogram.observe(histogramLabels, contentLength); + } }; diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index 5ac9594d8cdeb..25019e64d3799 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -427,9 +427,19 @@ const Api = new WebHookAPI({ }); Api.router - .use(loggerMiddleware(integrationLogger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi })) - .use(tracerSpanMiddleware); + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/hooks\//), + api: Api, + settings, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) + .use(tracerSpanMiddleware) + .use(loggerMiddleware(integrationLogger)); Api.addRoute( ':integrationId/:userId/:token', diff --git a/apps/meteor/app/lib/server/functions/notifications/desktop.ts b/apps/meteor/app/lib/server/functions/notifications/desktop.ts index 9b49f53582063..4554656d408b6 100644 --- a/apps/meteor/app/lib/server/functions/notifications/desktop.ts +++ b/apps/meteor/app/lib/server/functions/notifications/desktop.ts @@ -67,6 +67,7 @@ export async function notifyDesktopUser({ }; metrics.notificationsSent.inc({ notification_type: 'desktop' }); + metrics.notificationsSentTotal.inc({ notification_type: 'desktop' }); void api.broadcast('notify.desktop', userId, payload); } diff --git a/apps/meteor/app/lib/server/functions/notifications/email.js b/apps/meteor/app/lib/server/functions/notifications/email.js index 7bdbd0d6e990a..4de543abb7dd6 100644 --- a/apps/meteor/app/lib/server/functions/notifications/email.js +++ b/apps/meteor/app/lib/server/functions/notifications/email.js @@ -178,13 +178,14 @@ export async function getEmailData({ message, receiver, sender, subscription, ro )}${message.tmid || message._id}@${replyto.split('@')[1]}`; } - metrics.notificationsSent.inc({ notification_type: 'email' }); return email; } export function sendEmailFromData(data) { - metrics.notificationsSent.inc({ notification_type: 'email' }); - return Mailer.send(data); + return Mailer.send(data).then(() => { + metrics.notificationsSent.inc({ notification_type: 'email' }); + metrics.notificationsSentTotal.inc({ notification_type: 'email' }); + }); } export function shouldNotifyEmail({ diff --git a/apps/meteor/app/lib/server/lib/debug.js b/apps/meteor/app/lib/server/lib/debug.js index 71daf1a4fbcfc..d4c129a0767ff 100644 --- a/apps/meteor/app/lib/server/lib/debug.js +++ b/apps/meteor/app/lib/server/lib/debug.js @@ -57,11 +57,14 @@ const wrapMethods = function (name, originalHandler, methodsMap) { const method = name === 'stream' ? `${name}:${originalArgs[0]}` : name; - const end = metrics.meteorMethods.startTimer({ + const meteorMethodLabels = { method, has_connection: this.connection != null, has_user: this.userId != null, - }); + }; + + const end = metrics.meteorMethods.startTimer(meteorMethodLabels); + const endHistogram = metrics.meteorMethodsSeconds.startTimer(meteorMethodLabels); logger.method({ method, @@ -84,6 +87,7 @@ const wrapMethods = function (name, originalHandler, methodsMap) { async () => { const result = await originalHandler.apply(this, originalArgs); end(); + endHistogram(); return result; }, ); @@ -115,10 +119,12 @@ Meteor.publish = function (name, func) { }); const end = metrics.meteorSubscriptions.startTimer({ subscription: name }); + const endHistogram = metrics.meteorSubscriptionsSeconds.startTimer({ subscription: name }); const originalReady = this.ready; this.ready = function () { end(); + endHistogram(); return originalReady.apply(this, args); }; diff --git a/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts b/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts index 5e4dab8c4f22e..0d1bff3250a4c 100644 --- a/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts +++ b/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts @@ -40,6 +40,7 @@ export const apiDeprecationLogger = ((logger) => { writeDeprecationHeader(res, 'endpoint-deprecation', message, version); metrics.deprecations.inc({ type: 'deprecation', kind: 'endpoint', name: endpoint }); + metrics.deprecationsTotal.inc({ type: 'deprecation', kind: 'endpoint', name: endpoint }); logger.warn({ msg: message, endpoint, version, info }); }, @@ -63,6 +64,7 @@ export const apiDeprecationLogger = ((logger) => { } metrics.deprecations.inc({ type: 'parameter-deprecation', kind: 'endpoint', name: endpoint, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'parameter-deprecation', kind: 'endpoint', name: endpoint, params: parameter }); writeDeprecationHeader(res, 'parameter-deprecation', message, version); @@ -92,6 +94,7 @@ export const apiDeprecationLogger = ((logger) => { compareVersions(version, message); metrics.deprecations.inc({ type: 'invalid-usage', kind: 'endpoint', name: endpoint, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'invalid-usage', kind: 'endpoint', name: endpoint, params: parameter }); writeDeprecationHeader(res, 'invalid-usage', message, version); @@ -116,6 +119,7 @@ export const methodDeprecationLogger = ((logger) => { } compareVersions(version, message); metrics.deprecations.inc({ type: 'deprecation', name: method, kind: 'method' }); + metrics.deprecationsTotal.inc({ type: 'deprecation', name: method, kind: 'method' }); logger.warn({ msg: message, method, version, replacement }); }, parameter: (method: string, parameter: string, version: DeprecationLoggerNextPlannedVersion) => { @@ -125,6 +129,7 @@ export const methodDeprecationLogger = ((logger) => { } metrics.deprecations.inc({ type: 'parameter-deprecation', name: method, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'parameter-deprecation', name: method, params: parameter }); compareVersions(version, message); logger.warn({ msg: message, method, parameter, version }); @@ -151,6 +156,7 @@ export const methodDeprecationLogger = ((logger) => { compareVersions(version, message); metrics.deprecations.inc({ type: 'invalid-usage', name: method, params: parameter, kind: 'method' }); + metrics.deprecationsTotal.inc({ type: 'invalid-usage', name: method, params: parameter, kind: 'method' }); logger.warn({ msg: message, method, parameter, version }); }, diff --git a/apps/meteor/app/lib/server/lib/processDirectEmail.ts b/apps/meteor/app/lib/server/lib/processDirectEmail.ts index e88b7d58468e4..e930add5716e8 100644 --- a/apps/meteor/app/lib/server/lib/processDirectEmail.ts +++ b/apps/meteor/app/lib/server/lib/processDirectEmail.ts @@ -109,8 +109,6 @@ export const processDirectEmail = async function (email: ParsedMail): Promise = { ts: tsDiff < 10000 ? ts : new Date(), msg, @@ -120,5 +118,10 @@ export const processDirectEmail = async function (email: ParsedMail): Promise async (reply, input) => { if (reply.allowed === false) { rateLimiterLog({ msg, reply, input }); - metrics.ddpRateLimitExceeded.inc({ + const rateLimitLabels = { limit_name: name, user_id: input.userId, client_address: input.clientAddress, type: input.type, name: input.name, connection_id: input.connectionId, - }); + }; + metrics.ddpRateLimitExceeded.inc(rateLimitLabels); + metrics.ddpRateLimitExceededTotal.inc(rateLimitLabels); // sleep before sending the error to slow down next requests if (slowDownRate > 0 && reply.numInvocationsExceeded) { await sleep(slowDownRate * reply.numInvocationsExceeded); diff --git a/apps/meteor/app/livechat/server/lib/webhooks.ts b/apps/meteor/app/livechat/server/lib/webhooks.ts index 661b428cc7cb8..de49a6bb0cae7 100644 --- a/apps/meteor/app/livechat/server/lib/webhooks.ts +++ b/apps/meteor/app/livechat/server/lib/webhooks.ts @@ -34,6 +34,7 @@ export async function sendRequest( if (result.status === 200) { metrics.totalLivechatWebhooksSuccess.inc(); + metrics.totalLivechatWebhooksSuccessTotal.inc(); await cb?.(result); return result; } @@ -46,10 +47,12 @@ export async function sendRequest( response: await result.text(), }); metrics.totalLivechatWebhooksFailures.inc(); + metrics.totalLivechatWebhooksFailuresTotal.inc(); return; } metrics.totalLivechatWebhooksFailures.inc(); + metrics.totalLivechatWebhooksFailuresTotal.inc(); throw new Error(await result.text()); } catch (err) { const retryAfter = timeout * 4; diff --git a/apps/meteor/app/metrics/server/lib/collectMetrics.ts b/apps/meteor/app/metrics/server/lib/collectMetrics.ts index 6a393bb406506..a720e9c5aff70 100644 --- a/apps/meteor/app/metrics/server/lib/collectMetrics.ts +++ b/apps/meteor/app/metrics/server/lib/collectMetrics.ts @@ -92,6 +92,7 @@ app.use('/metrics', (_req, res) => { .metrics() .then((data) => { metrics.metricsRequests.inc(); + metrics.metricsRequestsTotal.inc(); metrics.metricsSize.set(data.length); res.end(data); diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index c35cd51f70ca6..939b2edbd0089 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -1,18 +1,33 @@ +import { performance } from 'node:perf_hooks'; + import client from 'prom-client'; const percentiles = [0.01, 0.1, 0.5, 0.9, 0.95, 0.99, 1]; +const latencyBuckets = [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0]; +const queueWaitBuckets = [1, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600]; + export const metrics = { deprecations: new client.Counter({ name: 'rocketchat_deprecations', labelNames: ['type', 'kind', 'name', 'params'], help: 'cumulated number of deprecations being used', }), + deprecationsTotal: new client.Counter({ + name: 'rocketchat_deprecations_total', + labelNames: ['type', 'kind', 'name', 'params'], + help: 'cumulated number of deprecations being used', + }), metricsRequests: new client.Counter({ name: 'rocketchat_metrics_requests', labelNames: ['notification_type'], help: 'cumulated number of calls to the metrics endpoint', }), + metricsRequestsTotal: new client.Counter({ + name: 'rocketchat_metrics_requests_total', + labelNames: ['notification_type'], + help: 'cumulated number of calls to the metrics endpoint', + }), metricsSize: new client.Gauge({ name: 'rocketchat_metrics_size', help: 'size of the metrics response in chars', @@ -28,24 +43,59 @@ export const metrics = { labelNames: ['method', 'has_connection', 'has_user'], percentiles, }), + meteorMethodsSeconds: new client.Histogram({ + name: 'rocketchat_meteor_methods_seconds', + help: 'histogram of meteor methods count and time in seconds', + labelNames: ['method', 'has_connection', 'has_user'], + buckets: latencyBuckets, + }), rocketchatCallbacks: new client.Summary({ name: 'rocketchat_callbacks', help: 'summary of rocketchat callbacks count and time', labelNames: ['hook', 'callback'], percentiles, }), + rocketchatCallbacksSeconds: new client.Histogram({ + name: 'rocketchat_callbacks_seconds', + help: 'histogram of rocketchat callbacks count and time in seconds', + labelNames: ['hook', 'callback'], + buckets: latencyBuckets, + }), rocketchatHooks: new client.Summary({ name: 'rocketchat_hooks', help: 'summary of rocketchat hooks count and time', labelNames: ['hook', 'callbacks_length'], percentiles, }), + rocketchatHooksSeconds: new client.Histogram({ + name: 'rocketchat_hooks_seconds', + help: 'histogram of rocketchat hooks count and time in seconds', + labelNames: ['hook', 'callbacks_length'], + buckets: latencyBuckets, + }), rocketchatRestApi: new client.Summary({ name: 'rocketchat_rest_api', help: 'summary of rocketchat rest api count and time', labelNames: ['method', 'entrypoint', 'user_agent', 'status', 'version'], percentiles, }), + rocketchatRestApiSeconds: new client.Histogram({ + name: 'rocketchat_rest_api_seconds', + help: 'histogram of rocketchat rest api count and time in seconds', + labelNames: ['method', 'entrypoint', 'status', 'version'], + buckets: latencyBuckets, + }), + rocketchatRestApiResponseSizeBytes: new client.Histogram({ + name: 'rocketchat_rest_api_response_size_bytes', + help: 'histogram of rocketchat rest api response sizes in bytes', + labelNames: ['method', 'entrypoint', 'status', 'version'], + buckets: [0, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000], + }), + rocketchatRestApiActiveRequests: new client.Gauge({ + name: 'rocketchat_rest_api_active_requests', + help: 'number of currently active rest api requests', + labelNames: ['method'], + }), meteorSubscriptions: new client.Summary({ name: 'rocketchat_meteor_subscriptions', @@ -53,16 +103,31 @@ export const metrics = { labelNames: ['subscription'], percentiles, }), + meteorSubscriptionsSeconds: new client.Histogram({ + name: 'rocketchat_meteor_subscriptions_seconds', + help: 'histogram of meteor subscriptions count and time in seconds', + labelNames: ['subscription'], + buckets: latencyBuckets, + }), messagesSent: new client.Counter({ name: 'rocketchat_message_sent', help: 'cumulated number of messages sent', }), + messagesSentTotal: new client.Counter({ + name: 'rocketchat_messages_sent_total', + help: 'cumulated number of messages sent', + }), notificationsSent: new client.Counter({ name: 'rocketchat_notification_sent', labelNames: ['notification_type'], help: 'cumulated number of notifications sent', }), + notificationsSentTotal: new client.Counter({ + name: 'rocketchat_notifications_sent_total', + labelNames: ['notification_type'], + help: 'cumulated number of notifications sent', + }), messageRoundtripTime: new client.Summary({ name: 'rocketchat_messages_roundtrip_time_summary', help: 'time spent by a message from save to receive back', @@ -71,6 +136,11 @@ export const metrics = { ageBuckets: 5, // pruneAgedBuckets: true, // Type not added to prom-client on 14.2 https://github.com/siimon/prom-client/pull/558 }), + messageRoundtripTimeSeconds: new client.Histogram({ + name: 'rocketchat_messages_roundtrip_time_seconds', + help: 'time in seconds spent by a message from save to receive back', + buckets: latencyBuckets, + }), ddpSessions: new client.Gauge({ name: 'rocketchat_ddp_sessions_count', @@ -89,6 +159,11 @@ export const metrics = { labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], help: 'number of times a ddp rate limiter was exceeded', }), + ddpRateLimitExceededTotal: new client.Counter({ + name: 'rocketchat_ddp_rate_limit_exceeded_total', + labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], + help: 'number of times a ddp rate limiter was exceeded', + }), version: new client.Gauge({ name: 'rocketchat_version', @@ -189,6 +264,12 @@ export const metrics = { labelNames: ['bridge', 'method', 'app_id'], percentiles, }), + appBridgeMethodsSeconds: new client.Histogram({ + name: 'rocketchat_apps_bridge_methods_seconds', + help: 'histogram of app bridge method calls count and time in seconds', + labelNames: ['bridge', 'method', 'app_id'], + buckets: latencyBuckets, + }), // Meteor Facts meteorFacts: new client.Gauge({ @@ -204,10 +285,18 @@ export const metrics = { name: 'rocketchat_livechat_webhooks_success', help: 'successful livechat webhooks', }), + totalLivechatWebhooksSuccessTotal: new client.Counter({ + name: 'rocketchat_livechat_webhooks_success_total', + help: 'successful livechat webhooks', + }), totalLivechatWebhooksFailures: new client.Counter({ name: 'rocketchat_livechat_webhooks_failures', help: 'failed livechat webhooks', }), + totalLivechatWebhooksFailuresTotal: new client.Counter({ + name: 'rocketchat_livechat_webhooks_failures_total', + help: 'failed livechat webhooks', + }), totalItemsProcessedByQueue: new client.Counter({ name: 'rocketchat_queue_items_processed_total', @@ -230,6 +319,22 @@ export const metrics = { help: 'Time taken in seconds for an item to be processed for the first time by Omni queues', percentiles, }), + timeToQueueProcessingByQueueHistogram: new client.Histogram({ + name: 'rocketchat_queue_wait_duration_seconds', + labelNames: ['queue'], + help: 'Histogram of time taken in seconds for an item to be processed for the first time by Omni queues', + buckets: queueWaitBuckets, + }), }; // Metrics +let eluBase = performance.eventLoopUtilization(); + +new client.Gauge({ + name: 'nodejs_event_loop_utilization_ratio', + help: 'Event Loop Utilization (ELU) as reported by NodeJS', + collect() { + this.set(performance.eventLoopUtilization(eluBase).utilization); + eluBase = performance.eventLoopUtilization(); + }, +}); diff --git a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts index 7fd7f404ac970..33c752825ae97 100644 --- a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts +++ b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts @@ -103,8 +103,10 @@ class PushNotification { idOnly, }); - metrics.notificationsSent.inc({ notification_type: 'mobile' }); await Push.send(config); + + metrics.notificationsSent.inc({ notification_type: 'mobile' }); + metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); } async getNotificationForMessageId({ diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index 27440ba7caeea..24e6e82d6c944 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -70,10 +70,21 @@ export class AppsRestApi { }); const logger = new Logger('APPS'); + this.api.router - .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi })) - .use(tracerSpanMiddleware); + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\/apps\//), + api: this.api, + settings, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) + .use(tracerSpanMiddleware) + .use(loggerMiddleware(logger)); this.addManagementRoutes(); // Using the same instance of the existing API for now, to be able to use the same api prefix(/api) diff --git a/apps/meteor/server/services/meteor/service.ts b/apps/meteor/server/services/meteor/service.ts index 7a4283e5020a2..f13b1152e50ff 100644 --- a/apps/meteor/server/services/meteor/service.ts +++ b/apps/meteor/server/services/meteor/service.ts @@ -140,7 +140,9 @@ export class MeteorService extends ServiceClassInternal implements IMeteor { if (!disableMsgRoundtripTracking) { this.onEvent('watch.messages', async ({ message }) => { if (message?._updatedAt instanceof Date) { - metrics.messageRoundtripTime.observe(Date.now() - message._updatedAt.getTime()); + const elapsedMs = Date.now() - message._updatedAt.getTime(); + metrics.messageRoundtripTime.observe(elapsedMs); + metrics.messageRoundtripTimeSeconds.observe(elapsedMs / 1000); } }); } diff --git a/apps/meteor/server/services/omnichannel/queue.ts b/apps/meteor/server/services/omnichannel/queue.ts index 68acf2401e961..8fd3de8586f01 100644 --- a/apps/meteor/server/services/omnichannel/queue.ts +++ b/apps/meteor/server/services/omnichannel/queue.ts @@ -269,7 +269,9 @@ export class OmnichannelQueue implements IOmnichannelQueue { void dispatchAgentDelegated(rid, agentId); }, 1000); - metrics.timeToQueueProcessingByQueue.observe({ queue }, (Date.now() - inquiry.ts.getTime()) / 1000); + const waitTimeSeconds = (Date.now() - inquiry.ts.getTime()) / 1000; + metrics.timeToQueueProcessingByQueue.observe({ queue }, waitTimeSeconds); + metrics.timeToQueueProcessingByQueueHistogram.observe({ queue }, waitTimeSeconds); metrics.totalItemsProcessedByQueue.inc({ queue }); return true; } diff --git a/apps/meteor/server/services/video-conference/service.ts b/apps/meteor/server/services/video-conference/service.ts index 1517dc4d56e0b..fc6080d68a6f7 100644 --- a/apps/meteor/server/services/video-conference/service.ts +++ b/apps/meteor/server/services/video-conference/service.ts @@ -674,7 +674,6 @@ export class VideoConfService extends ServiceClassInternal implements IVideoConf return; } - metrics.notificationsSent.inc({ notification_type: 'mobile' }); await Push.send({ from: 'push', badge: 0, @@ -701,6 +700,9 @@ export class VideoConfService extends ServiceClassInternal implements IVideoConf category: 'VIDEOCONF', }, }); + + metrics.notificationsSent.inc({ notification_type: 'mobile' }); + metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); } private async sendAllPushNotifications(callId: VideoConference['_id']): Promise { diff --git a/apps/meteor/server/startup/callbacks.ts b/apps/meteor/server/startup/callbacks.ts index d252e9555f3bd..ac42ed58137b0 100644 --- a/apps/meteor/server/startup/callbacks.ts +++ b/apps/meteor/server/startup/callbacks.ts @@ -11,18 +11,28 @@ callbacks.setMetricsTrackers({ const start = performance.now(); const stopTimer = metrics.rocketchatCallbacks.startTimer({ hook, callback: id }); + const stopHistogram = metrics.rocketchatCallbacksSeconds.startTimer({ hook, callback: id }); return (): void => { const end = performance.now(); StatsTracker.timing('callbacks.time', end - start, [`hook:${hook}`, `callback:${id}`]); stopTimer(); + stopHistogram(); }; }, trackHook: ({ hook, length }) => { - return metrics.rocketchatHooks.startTimer({ + const stopTimer = metrics.rocketchatHooks.startTimer({ hook, callbacks_length: length, }); + const stopHistogram = metrics.rocketchatHooksSeconds.startTimer({ + hook, + callbacks_length: length, + }); + return (): void => { + stopTimer(); + stopHistogram(); + }; }, }); diff --git a/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts b/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts index 6fc16513f1de5..6266a13fdcb68 100644 --- a/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts +++ b/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts @@ -50,7 +50,9 @@ function buildSubject(options?: { const metrics = { totalLivechatWebhooksSuccess: { inc: sinon.spy() }, + totalLivechatWebhooksSuccessTotal: { inc: sinon.spy() }, totalLivechatWebhooksFailures: { inc: sinon.spy() }, + totalLivechatWebhooksFailuresTotal: { inc: sinon.spy() }, }; const { sendRequest } = proxyquire.noCallThru().load(MODULE_PATH, { diff --git a/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts b/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts index 1580620ae509f..470b79daa8fa3 100644 --- a/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts +++ b/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts @@ -60,6 +60,7 @@ const { OmnichannelQueue } = p.noCallThru().load('../../../../../server/services '../../../app/metrics/server': { metrics: { timeToQueueProcessingByQueue: { observe: Sinon.stub() }, + timeToQueueProcessingByQueueHistogram: { observe: Sinon.stub() }, totalItemsProcessedByQueue: { inc: Sinon.stub() }, totalItemsProcessedByReconciliationQueue: { inc: Sinon.stub() }, totalItemsFailedByQueue: { inc: Sinon.stub() },