From 9d755c455b186f1df2e84fa72c4cfc3c1cf932b7 Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Thu, 26 Feb 2026 12:18:11 -0300 Subject: [PATCH 01/19] add eventLoopUtilization to metrics --- apps/meteor/app/metrics/server/lib/metrics.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index c35cd51f70ca6..6a8ebc88004cc 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -1,3 +1,5 @@ +import { performance } from 'node:perf_hooks'; + import client from 'prom-client'; const percentiles = [0.01, 0.1, 0.5, 0.9, 0.95, 0.99, 1]; @@ -233,3 +235,13 @@ export const metrics = { }; // Metrics +let eluBase = performance.eventLoopUtilization(); + +new client.Gauge({ + name: 'nodejs_event_loop_utilization', + help: 'Event Loop Utilization (ELU) as reported by Node', + collect() { + this.set(performance.eventLoopUtilization(eluBase).utilization); + eluBase = performance.eventLoopUtilization(); + }, +}); From 1398fc202d381c6c10c9764016b84599db34e6a2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:52:13 +0000 Subject: [PATCH 02/19] add Prometheus best-practices metric duplicates (Histograms + _total counters)" Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/api.ts | 2 +- .../app/api/server/middlewares/metrics.ts | 12 ++- .../meteor/app/integrations/server/api/api.ts | 2 +- .../server/functions/notifications/desktop.ts | 1 + .../server/functions/notifications/email.js | 2 + apps/meteor/app/lib/server/lib/debug.js | 10 ++- .../server/lib/deprecationWarningLogger.ts | 6 ++ .../app/lib/server/lib/processDirectEmail.ts | 1 + .../app/lib/server/methods/sendMessage.ts | 1 + .../app/lib/server/startup/rateLimiter.js | 8 ++ .../app/livechat/server/lib/webhooks.ts | 3 + .../app/metrics/server/lib/collectMetrics.ts | 1 + apps/meteor/app/metrics/server/lib/metrics.ts | 82 +++++++++++++++++++ .../server/lib/PushNotification.ts | 1 + .../ee/server/apps/communication/rest.ts | 2 +- apps/meteor/server/services/meteor/service.ts | 4 +- .../server/services/omnichannel/queue.ts | 4 +- .../services/video-conference/service.ts | 1 + apps/meteor/server/startup/callbacks.ts | 12 ++- .../app/livechat/server/lib/webhooks.spec.ts | 2 + .../services/omnichannel/queue.tests.ts | 1 + 21 files changed, 147 insertions(+), 11 deletions(-) diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index dbafa020706f8..a79e359519aa4 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -105,7 +105,7 @@ export const startRestAPI = () => { .use(remoteAddressMiddleware) .use(cors(settings)) .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi })) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) .use(tracerSpanMiddleware) .use(API.v1.router) .use(API.default.router).router, diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index bedc2cc823a93..8007900d83962 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -1,5 +1,5 @@ import type { MiddlewareHandler } from 'hono'; -import type { Summary } from 'prom-client'; +import type { Histogram, Summary } from 'prom-client'; import type { CachedSettings } from '../../../settings/server/CachedSettings'; import type { APIClass } from '../ApiClass'; @@ -10,14 +10,17 @@ export const metricsMiddleware = api, settings, summary, + histogram, }: { basePathRegex?: RegExp; api: APIClass; settings: CachedSettings; summary: Summary; + histogram?: Histogram; }): MiddlewareHandler => async (c, next) => { const rocketchatRestApiEnd = summary.startTimer(); + const rocketchatRestApiHistEnd = histogram?.startTimer(); await next(); @@ -26,11 +29,14 @@ export const metricsMiddleware = // get rid of the base path (i.e.: /api/v1/) const entrypoint = basePathRegex ? routePath.replace(basePathRegex, '') : routePath; - rocketchatRestApiEnd({ + const labels = { status: c.res.status, method: method.toLowerCase(), version: api.version, ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), entrypoint: basePathRegex && entrypoint.startsWith('method.call') ? decodeURIComponent(path.replace(basePathRegex, '')) : entrypoint, - }); + }; + + rocketchatRestApiEnd(labels); + rocketchatRestApiHistEnd?.(labels); }; diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index 5ac9594d8cdeb..8a81ef5f2615b 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -428,7 +428,7 @@ const Api = new WebHookAPI({ Api.router .use(loggerMiddleware(integrationLogger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi })) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) .use(tracerSpanMiddleware); Api.addRoute( diff --git a/apps/meteor/app/lib/server/functions/notifications/desktop.ts b/apps/meteor/app/lib/server/functions/notifications/desktop.ts index 9b49f53582063..4554656d408b6 100644 --- a/apps/meteor/app/lib/server/functions/notifications/desktop.ts +++ b/apps/meteor/app/lib/server/functions/notifications/desktop.ts @@ -67,6 +67,7 @@ export async function notifyDesktopUser({ }; metrics.notificationsSent.inc({ notification_type: 'desktop' }); + metrics.notificationsSentTotal.inc({ notification_type: 'desktop' }); void api.broadcast('notify.desktop', userId, payload); } diff --git a/apps/meteor/app/lib/server/functions/notifications/email.js b/apps/meteor/app/lib/server/functions/notifications/email.js index 7bdbd0d6e990a..6c92dee3b2bc1 100644 --- a/apps/meteor/app/lib/server/functions/notifications/email.js +++ b/apps/meteor/app/lib/server/functions/notifications/email.js @@ -179,11 +179,13 @@ export async function getEmailData({ message, receiver, sender, subscription, ro } metrics.notificationsSent.inc({ notification_type: 'email' }); + metrics.notificationsSentTotal.inc({ notification_type: 'email' }); return email; } export function sendEmailFromData(data) { metrics.notificationsSent.inc({ notification_type: 'email' }); + metrics.notificationsSentTotal.inc({ notification_type: 'email' }); return Mailer.send(data); } diff --git a/apps/meteor/app/lib/server/lib/debug.js b/apps/meteor/app/lib/server/lib/debug.js index 71daf1a4fbcfc..d4c129a0767ff 100644 --- a/apps/meteor/app/lib/server/lib/debug.js +++ b/apps/meteor/app/lib/server/lib/debug.js @@ -57,11 +57,14 @@ const wrapMethods = function (name, originalHandler, methodsMap) { const method = name === 'stream' ? `${name}:${originalArgs[0]}` : name; - const end = metrics.meteorMethods.startTimer({ + const meteorMethodLabels = { method, has_connection: this.connection != null, has_user: this.userId != null, - }); + }; + + const end = metrics.meteorMethods.startTimer(meteorMethodLabels); + const endHistogram = metrics.meteorMethodsSeconds.startTimer(meteorMethodLabels); logger.method({ method, @@ -84,6 +87,7 @@ const wrapMethods = function (name, originalHandler, methodsMap) { async () => { const result = await originalHandler.apply(this, originalArgs); end(); + endHistogram(); return result; }, ); @@ -115,10 +119,12 @@ Meteor.publish = function (name, func) { }); const end = metrics.meteorSubscriptions.startTimer({ subscription: name }); + const endHistogram = metrics.meteorSubscriptionsSeconds.startTimer({ subscription: name }); const originalReady = this.ready; this.ready = function () { end(); + endHistogram(); return originalReady.apply(this, args); }; diff --git a/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts b/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts index 5e4dab8c4f22e..0d1bff3250a4c 100644 --- a/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts +++ b/apps/meteor/app/lib/server/lib/deprecationWarningLogger.ts @@ -40,6 +40,7 @@ export const apiDeprecationLogger = ((logger) => { writeDeprecationHeader(res, 'endpoint-deprecation', message, version); metrics.deprecations.inc({ type: 'deprecation', kind: 'endpoint', name: endpoint }); + metrics.deprecationsTotal.inc({ type: 'deprecation', kind: 'endpoint', name: endpoint }); logger.warn({ msg: message, endpoint, version, info }); }, @@ -63,6 +64,7 @@ export const apiDeprecationLogger = ((logger) => { } metrics.deprecations.inc({ type: 'parameter-deprecation', kind: 'endpoint', name: endpoint, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'parameter-deprecation', kind: 'endpoint', name: endpoint, params: parameter }); writeDeprecationHeader(res, 'parameter-deprecation', message, version); @@ -92,6 +94,7 @@ export const apiDeprecationLogger = ((logger) => { compareVersions(version, message); metrics.deprecations.inc({ type: 'invalid-usage', kind: 'endpoint', name: endpoint, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'invalid-usage', kind: 'endpoint', name: endpoint, params: parameter }); writeDeprecationHeader(res, 'invalid-usage', message, version); @@ -116,6 +119,7 @@ export const methodDeprecationLogger = ((logger) => { } compareVersions(version, message); metrics.deprecations.inc({ type: 'deprecation', name: method, kind: 'method' }); + metrics.deprecationsTotal.inc({ type: 'deprecation', name: method, kind: 'method' }); logger.warn({ msg: message, method, version, replacement }); }, parameter: (method: string, parameter: string, version: DeprecationLoggerNextPlannedVersion) => { @@ -125,6 +129,7 @@ export const methodDeprecationLogger = ((logger) => { } metrics.deprecations.inc({ type: 'parameter-deprecation', name: method, params: parameter }); + metrics.deprecationsTotal.inc({ type: 'parameter-deprecation', name: method, params: parameter }); compareVersions(version, message); logger.warn({ msg: message, method, parameter, version }); @@ -151,6 +156,7 @@ export const methodDeprecationLogger = ((logger) => { compareVersions(version, message); metrics.deprecations.inc({ type: 'invalid-usage', name: method, params: parameter, kind: 'method' }); + metrics.deprecationsTotal.inc({ type: 'invalid-usage', name: method, params: parameter, kind: 'method' }); logger.warn({ msg: message, method, parameter, version }); }, diff --git a/apps/meteor/app/lib/server/lib/processDirectEmail.ts b/apps/meteor/app/lib/server/lib/processDirectEmail.ts index e88b7d58468e4..af4a1ebad7fe2 100644 --- a/apps/meteor/app/lib/server/lib/processDirectEmail.ts +++ b/apps/meteor/app/lib/server/lib/processDirectEmail.ts @@ -110,6 +110,7 @@ export const processDirectEmail = async function (email: ParsedMail): Promise = { ts: tsDiff < 10000 ? ts : new Date(), diff --git a/apps/meteor/app/lib/server/methods/sendMessage.ts b/apps/meteor/app/lib/server/methods/sendMessage.ts index a1208543d6841..f6fdcd748113c 100644 --- a/apps/meteor/app/lib/server/methods/sendMessage.ts +++ b/apps/meteor/app/lib/server/methods/sendMessage.ts @@ -106,6 +106,7 @@ export async function executeSendMessage( } metrics.messagesSent.inc(); // TODO This line needs to be moved to it's proper place. See the comments on: https://github.com/RocketChat/Rocket.Chat/pull/5736 + metrics.messagesSentTotal.inc(); return await sendMessage(user, message, room, { previewUrls: extraInfo?.previewUrls }); } catch (err: any) { SystemLogger.error({ msg: 'Error sending message:', err }); diff --git a/apps/meteor/app/lib/server/startup/rateLimiter.js b/apps/meteor/app/lib/server/startup/rateLimiter.js index 5a312f4520d46..a5784c702cbec 100644 --- a/apps/meteor/app/lib/server/startup/rateLimiter.js +++ b/apps/meteor/app/lib/server/startup/rateLimiter.js @@ -134,6 +134,14 @@ const callback = (msg, name) => async (reply, input) => { name: input.name, connection_id: input.connectionId, }); + metrics.ddpRateLimitExceededTotal.inc({ + limit_name: name, + user_id: input.userId, + client_address: input.clientAddress, + type: input.type, + name: input.name, + connection_id: input.connectionId, + }); // sleep before sending the error to slow down next requests if (slowDownRate > 0 && reply.numInvocationsExceeded) { await sleep(slowDownRate * reply.numInvocationsExceeded); diff --git a/apps/meteor/app/livechat/server/lib/webhooks.ts b/apps/meteor/app/livechat/server/lib/webhooks.ts index 661b428cc7cb8..de49a6bb0cae7 100644 --- a/apps/meteor/app/livechat/server/lib/webhooks.ts +++ b/apps/meteor/app/livechat/server/lib/webhooks.ts @@ -34,6 +34,7 @@ export async function sendRequest( if (result.status === 200) { metrics.totalLivechatWebhooksSuccess.inc(); + metrics.totalLivechatWebhooksSuccessTotal.inc(); await cb?.(result); return result; } @@ -46,10 +47,12 @@ export async function sendRequest( response: await result.text(), }); metrics.totalLivechatWebhooksFailures.inc(); + metrics.totalLivechatWebhooksFailuresTotal.inc(); return; } metrics.totalLivechatWebhooksFailures.inc(); + metrics.totalLivechatWebhooksFailuresTotal.inc(); throw new Error(await result.text()); } catch (err) { const retryAfter = timeout * 4; diff --git a/apps/meteor/app/metrics/server/lib/collectMetrics.ts b/apps/meteor/app/metrics/server/lib/collectMetrics.ts index 6a393bb406506..a720e9c5aff70 100644 --- a/apps/meteor/app/metrics/server/lib/collectMetrics.ts +++ b/apps/meteor/app/metrics/server/lib/collectMetrics.ts @@ -92,6 +92,7 @@ app.use('/metrics', (_req, res) => { .metrics() .then((data) => { metrics.metricsRequests.inc(); + metrics.metricsRequestsTotal.inc(); metrics.metricsSize.set(data.length); res.end(data); diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index 6a8ebc88004cc..7a0e64902860c 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -4,17 +4,30 @@ import client from 'prom-client'; const percentiles = [0.01, 0.1, 0.5, 0.9, 0.95, 0.99, 1]; +const latencyBuckets = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; +const queueWaitBuckets = [1, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600]; + export const metrics = { deprecations: new client.Counter({ name: 'rocketchat_deprecations', labelNames: ['type', 'kind', 'name', 'params'], help: 'cumulated number of deprecations being used', }), + deprecationsTotal: new client.Counter({ + name: 'rocketchat_deprecations_total', + labelNames: ['type', 'kind', 'name', 'params'], + help: 'cumulated number of deprecations being used', + }), metricsRequests: new client.Counter({ name: 'rocketchat_metrics_requests', labelNames: ['notification_type'], help: 'cumulated number of calls to the metrics endpoint', }), + metricsRequestsTotal: new client.Counter({ + name: 'rocketchat_metrics_requests_total', + labelNames: ['notification_type'], + help: 'cumulated number of calls to the metrics endpoint', + }), metricsSize: new client.Gauge({ name: 'rocketchat_metrics_size', help: 'size of the metrics response in chars', @@ -30,24 +43,48 @@ export const metrics = { labelNames: ['method', 'has_connection', 'has_user'], percentiles, }), + meteorMethodsSeconds: new client.Histogram({ + name: 'rocketchat_meteor_methods_seconds', + help: 'histogram of meteor methods count and time in seconds', + labelNames: ['method', 'has_connection', 'has_user'], + buckets: latencyBuckets, + }), rocketchatCallbacks: new client.Summary({ name: 'rocketchat_callbacks', help: 'summary of rocketchat callbacks count and time', labelNames: ['hook', 'callback'], percentiles, }), + rocketchatCallbacksSeconds: new client.Histogram({ + name: 'rocketchat_callbacks_seconds', + help: 'histogram of rocketchat callbacks count and time in seconds', + labelNames: ['hook', 'callback'], + buckets: latencyBuckets, + }), rocketchatHooks: new client.Summary({ name: 'rocketchat_hooks', help: 'summary of rocketchat hooks count and time', labelNames: ['hook', 'callbacks_length'], percentiles, }), + rocketchatHooksSeconds: new client.Histogram({ + name: 'rocketchat_hooks_seconds', + help: 'histogram of rocketchat hooks count and time in seconds', + labelNames: ['hook', 'callbacks_length'], + buckets: latencyBuckets, + }), rocketchatRestApi: new client.Summary({ name: 'rocketchat_rest_api', help: 'summary of rocketchat rest api count and time', labelNames: ['method', 'entrypoint', 'user_agent', 'status', 'version'], percentiles, }), + rocketchatRestApiSeconds: new client.Histogram({ + name: 'rocketchat_rest_api_seconds', + help: 'histogram of rocketchat rest api count and time in seconds', + labelNames: ['method', 'entrypoint', 'user_agent', 'status', 'version'], + buckets: latencyBuckets, + }), meteorSubscriptions: new client.Summary({ name: 'rocketchat_meteor_subscriptions', @@ -55,16 +92,31 @@ export const metrics = { labelNames: ['subscription'], percentiles, }), + meteorSubscriptionsSeconds: new client.Histogram({ + name: 'rocketchat_meteor_subscriptions_seconds', + help: 'histogram of meteor subscriptions count and time in seconds', + labelNames: ['subscription'], + buckets: latencyBuckets, + }), messagesSent: new client.Counter({ name: 'rocketchat_message_sent', help: 'cumulated number of messages sent', }), + messagesSentTotal: new client.Counter({ + name: 'rocketchat_messages_sent_total', + help: 'cumulated number of messages sent', + }), notificationsSent: new client.Counter({ name: 'rocketchat_notification_sent', labelNames: ['notification_type'], help: 'cumulated number of notifications sent', }), + notificationsSentTotal: new client.Counter({ + name: 'rocketchat_notifications_sent_total', + labelNames: ['notification_type'], + help: 'cumulated number of notifications sent', + }), messageRoundtripTime: new client.Summary({ name: 'rocketchat_messages_roundtrip_time_summary', help: 'time spent by a message from save to receive back', @@ -73,6 +125,11 @@ export const metrics = { ageBuckets: 5, // pruneAgedBuckets: true, // Type not added to prom-client on 14.2 https://github.com/siimon/prom-client/pull/558 }), + messageRoundtripTimeSeconds: new client.Histogram({ + name: 'rocketchat_messages_roundtrip_time_seconds', + help: 'time in seconds spent by a message from save to receive back', + buckets: latencyBuckets, + }), ddpSessions: new client.Gauge({ name: 'rocketchat_ddp_sessions_count', @@ -91,6 +148,11 @@ export const metrics = { labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], help: 'number of times a ddp rate limiter was exceeded', }), + ddpRateLimitExceededTotal: new client.Counter({ + name: 'rocketchat_ddp_rate_limit_exceeded_total', + labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], + help: 'number of times a ddp rate limiter was exceeded', + }), version: new client.Gauge({ name: 'rocketchat_version', @@ -191,6 +253,12 @@ export const metrics = { labelNames: ['bridge', 'method', 'app_id'], percentiles, }), + appBridgeMethodsSeconds: new client.Histogram({ + name: 'rocketchat_apps_bridge_methods_seconds', + help: 'histogram of app bridge method calls count and time in seconds', + labelNames: ['bridge', 'method', 'app_id'], + buckets: latencyBuckets, + }), // Meteor Facts meteorFacts: new client.Gauge({ @@ -206,10 +274,18 @@ export const metrics = { name: 'rocketchat_livechat_webhooks_success', help: 'successful livechat webhooks', }), + totalLivechatWebhooksSuccessTotal: new client.Counter({ + name: 'rocketchat_livechat_webhooks_success_total', + help: 'successful livechat webhooks', + }), totalLivechatWebhooksFailures: new client.Counter({ name: 'rocketchat_livechat_webhooks_failures', help: 'failed livechat webhooks', }), + totalLivechatWebhooksFailuresTotal: new client.Counter({ + name: 'rocketchat_livechat_webhooks_failures_total', + help: 'failed livechat webhooks', + }), totalItemsProcessedByQueue: new client.Counter({ name: 'rocketchat_queue_items_processed_total', @@ -232,6 +308,12 @@ export const metrics = { help: 'Time taken in seconds for an item to be processed for the first time by Omni queues', percentiles, }), + timeToQueueProcessingByQueueHistogram: new client.Histogram({ + name: 'rocketchat_queue_wait_duration_seconds', + labelNames: ['queue'], + help: 'Histogram of time taken in seconds for an item to be processed for the first time by Omni queues', + buckets: queueWaitBuckets, + }), }; // Metrics diff --git a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts index 7fd7f404ac970..13ef97a198ad9 100644 --- a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts +++ b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts @@ -104,6 +104,7 @@ class PushNotification { }); metrics.notificationsSent.inc({ notification_type: 'mobile' }); + metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); await Push.send(config); } diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index 27440ba7caeea..0c6460fb31fe9 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -72,7 +72,7 @@ export class AppsRestApi { const logger = new Logger('APPS'); this.api.router .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi })) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) .use(tracerSpanMiddleware); this.addManagementRoutes(); diff --git a/apps/meteor/server/services/meteor/service.ts b/apps/meteor/server/services/meteor/service.ts index 7a4283e5020a2..f13b1152e50ff 100644 --- a/apps/meteor/server/services/meteor/service.ts +++ b/apps/meteor/server/services/meteor/service.ts @@ -140,7 +140,9 @@ export class MeteorService extends ServiceClassInternal implements IMeteor { if (!disableMsgRoundtripTracking) { this.onEvent('watch.messages', async ({ message }) => { if (message?._updatedAt instanceof Date) { - metrics.messageRoundtripTime.observe(Date.now() - message._updatedAt.getTime()); + const elapsedMs = Date.now() - message._updatedAt.getTime(); + metrics.messageRoundtripTime.observe(elapsedMs); + metrics.messageRoundtripTimeSeconds.observe(elapsedMs / 1000); } }); } diff --git a/apps/meteor/server/services/omnichannel/queue.ts b/apps/meteor/server/services/omnichannel/queue.ts index 68acf2401e961..8fd3de8586f01 100644 --- a/apps/meteor/server/services/omnichannel/queue.ts +++ b/apps/meteor/server/services/omnichannel/queue.ts @@ -269,7 +269,9 @@ export class OmnichannelQueue implements IOmnichannelQueue { void dispatchAgentDelegated(rid, agentId); }, 1000); - metrics.timeToQueueProcessingByQueue.observe({ queue }, (Date.now() - inquiry.ts.getTime()) / 1000); + const waitTimeSeconds = (Date.now() - inquiry.ts.getTime()) / 1000; + metrics.timeToQueueProcessingByQueue.observe({ queue }, waitTimeSeconds); + metrics.timeToQueueProcessingByQueueHistogram.observe({ queue }, waitTimeSeconds); metrics.totalItemsProcessedByQueue.inc({ queue }); return true; } diff --git a/apps/meteor/server/services/video-conference/service.ts b/apps/meteor/server/services/video-conference/service.ts index 1517dc4d56e0b..ac0d287b8fa0b 100644 --- a/apps/meteor/server/services/video-conference/service.ts +++ b/apps/meteor/server/services/video-conference/service.ts @@ -675,6 +675,7 @@ export class VideoConfService extends ServiceClassInternal implements IVideoConf } metrics.notificationsSent.inc({ notification_type: 'mobile' }); + metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); await Push.send({ from: 'push', badge: 0, diff --git a/apps/meteor/server/startup/callbacks.ts b/apps/meteor/server/startup/callbacks.ts index d252e9555f3bd..ac42ed58137b0 100644 --- a/apps/meteor/server/startup/callbacks.ts +++ b/apps/meteor/server/startup/callbacks.ts @@ -11,18 +11,28 @@ callbacks.setMetricsTrackers({ const start = performance.now(); const stopTimer = metrics.rocketchatCallbacks.startTimer({ hook, callback: id }); + const stopHistogram = metrics.rocketchatCallbacksSeconds.startTimer({ hook, callback: id }); return (): void => { const end = performance.now(); StatsTracker.timing('callbacks.time', end - start, [`hook:${hook}`, `callback:${id}`]); stopTimer(); + stopHistogram(); }; }, trackHook: ({ hook, length }) => { - return metrics.rocketchatHooks.startTimer({ + const stopTimer = metrics.rocketchatHooks.startTimer({ hook, callbacks_length: length, }); + const stopHistogram = metrics.rocketchatHooksSeconds.startTimer({ + hook, + callbacks_length: length, + }); + return (): void => { + stopTimer(); + stopHistogram(); + }; }, }); diff --git a/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts b/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts index 6fc16513f1de5..6266a13fdcb68 100644 --- a/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts +++ b/apps/meteor/tests/unit/app/livechat/server/lib/webhooks.spec.ts @@ -50,7 +50,9 @@ function buildSubject(options?: { const metrics = { totalLivechatWebhooksSuccess: { inc: sinon.spy() }, + totalLivechatWebhooksSuccessTotal: { inc: sinon.spy() }, totalLivechatWebhooksFailures: { inc: sinon.spy() }, + totalLivechatWebhooksFailuresTotal: { inc: sinon.spy() }, }; const { sendRequest } = proxyquire.noCallThru().load(MODULE_PATH, { diff --git a/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts b/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts index 1580620ae509f..470b79daa8fa3 100644 --- a/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts +++ b/apps/meteor/tests/unit/server/services/omnichannel/queue.tests.ts @@ -60,6 +60,7 @@ const { OmnichannelQueue } = p.noCallThru().load('../../../../../server/services '../../../app/metrics/server': { metrics: { timeToQueueProcessingByQueue: { observe: Sinon.stub() }, + timeToQueueProcessingByQueueHistogram: { observe: Sinon.stub() }, totalItemsProcessedByQueue: { inc: Sinon.stub() }, totalItemsProcessedByReconciliationQueue: { inc: Sinon.stub() }, totalItemsFailedByQueue: { inc: Sinon.stub() }, From d9c436bb012519f87486ce1992e390d7647afebc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:38:22 +0000 Subject: [PATCH 03/19] remove user_agent and version labels from rocketchatRestApiSeconds histogram Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/middlewares/metrics.ts | 12 +++++++----- apps/meteor/app/metrics/server/lib/metrics.ts | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index 8007900d83962..0ae537701a5f5 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -29,14 +29,16 @@ export const metricsMiddleware = // get rid of the base path (i.e.: /api/v1/) const entrypoint = basePathRegex ? routePath.replace(basePathRegex, '') : routePath; - const labels = { + const histogramLabels = { status: c.res.status, method: method.toLowerCase(), - version: api.version, - ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), entrypoint: basePathRegex && entrypoint.startsWith('method.call') ? decodeURIComponent(path.replace(basePathRegex, '')) : entrypoint, }; - rocketchatRestApiEnd(labels); - rocketchatRestApiHistEnd?.(labels); + rocketchatRestApiEnd({ + ...histogramLabels, + version: api.version, + ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), + }); + rocketchatRestApiHistEnd?.(histogramLabels); }; diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index 7a0e64902860c..9293ac0655f0e 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -82,7 +82,7 @@ export const metrics = { rocketchatRestApiSeconds: new client.Histogram({ name: 'rocketchat_rest_api_seconds', help: 'histogram of rocketchat rest api count and time in seconds', - labelNames: ['method', 'entrypoint', 'user_agent', 'status', 'version'], + labelNames: ['method', 'entrypoint', 'status'], buckets: latencyBuckets, }), From 63bec0663aa53510c7a64c62d81816fe1b696072 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:38:47 +0000 Subject: [PATCH 04/19] make histogram non-optional in metricsMiddleware Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/middlewares/metrics.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index 0ae537701a5f5..73a4cca68080d 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -16,11 +16,11 @@ export const metricsMiddleware = api: APIClass; settings: CachedSettings; summary: Summary; - histogram?: Histogram; + histogram: Histogram; }): MiddlewareHandler => async (c, next) => { const rocketchatRestApiEnd = summary.startTimer(); - const rocketchatRestApiHistEnd = histogram?.startTimer(); + const rocketchatRestApiHistEnd = histogram.startTimer(); await next(); @@ -40,5 +40,5 @@ export const metricsMiddleware = version: api.version, ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), }); - rocketchatRestApiHistEnd?.(histogramLabels); + rocketchatRestApiHistEnd(histogramLabels); }; From 0fa52a86f5771c740c33d5d8b6bee0d3e58a514d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:39:08 +0000 Subject: [PATCH 05/19] extract rate limit labels to variable in rateLimiter.js Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/lib/server/startup/rateLimiter.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/apps/meteor/app/lib/server/startup/rateLimiter.js b/apps/meteor/app/lib/server/startup/rateLimiter.js index a5784c702cbec..d493eccfb4b39 100644 --- a/apps/meteor/app/lib/server/startup/rateLimiter.js +++ b/apps/meteor/app/lib/server/startup/rateLimiter.js @@ -126,22 +126,16 @@ const ruleIds = {}; const callback = (msg, name) => async (reply, input) => { if (reply.allowed === false) { rateLimiterLog({ msg, reply, input }); - metrics.ddpRateLimitExceeded.inc({ + const rateLimitLabels = { limit_name: name, user_id: input.userId, client_address: input.clientAddress, type: input.type, name: input.name, connection_id: input.connectionId, - }); - metrics.ddpRateLimitExceededTotal.inc({ - limit_name: name, - user_id: input.userId, - client_address: input.clientAddress, - type: input.type, - name: input.name, - connection_id: input.connectionId, - }); + }; + metrics.ddpRateLimitExceeded.inc(rateLimitLabels); + metrics.ddpRateLimitExceededTotal.inc(rateLimitLabels); // sleep before sending the error to slow down next requests if (slowDownRate > 0 && reply.numInvocationsExceeded) { await sleep(slowDownRate * reply.numInvocationsExceeded); From 9919625bdcbefb2371f22ec6f9134f9ec01eb474 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:39:38 +0000 Subject: [PATCH 06/19] move metrics middleware first and tracer second in api.ts middleware chain Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index a79e359519aa4..691d1000695a9 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -102,11 +102,11 @@ settings.watch('API_Enable_Rate_Limiter_Limit_Calls_Default', (value) => export const startRestAPI = () => { (WebApp.rawConnectHandlers as unknown as ReturnType).use( API.api + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) + .use(tracerSpanMiddleware) .use(remoteAddressMiddleware) .use(cors(settings)) .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) - .use(tracerSpanMiddleware) .use(API.v1.router) .use(API.default.router).router, ); From 9dd7e770ba6f2af332f7a870ede47845383d3716 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:40:01 +0000 Subject: [PATCH 07/19] move metrics middleware to end of chain in integrations api Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/integrations/server/api/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index 8a81ef5f2615b..c40ae48ee13ed 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -428,8 +428,8 @@ const Api = new WebHookAPI({ Api.router .use(loggerMiddleware(integrationLogger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) - .use(tracerSpanMiddleware); + .use(tracerSpanMiddleware) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })); Api.addRoute( ':integrationId/:userId/:token', From c8a0acc4c26db8094beaa3143465b6cbb487cb0a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:41:16 +0000 Subject: [PATCH 08/19] add response size histogram and active requests gauge to metrics middleware Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/api.ts | 2 +- .../meteor/app/api/server/middlewares/metrics.ts | 16 +++++++++++++++- apps/meteor/app/integrations/server/api/api.ts | 2 +- apps/meteor/app/metrics/server/lib/metrics.ts | 11 +++++++++++ apps/meteor/ee/server/apps/communication/rest.ts | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index 691d1000695a9..cb95019294545 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -102,7 +102,7 @@ settings.watch('API_Enable_Rate_Limiter_Limit_Calls_Default', (value) => export const startRestAPI = () => { (WebApp.rawConnectHandlers as unknown as ReturnType).use( API.api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) .use(tracerSpanMiddleware) .use(remoteAddressMiddleware) .use(cors(settings)) diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index 73a4cca68080d..e8423b885b8bb 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -1,5 +1,5 @@ import type { MiddlewareHandler } from 'hono'; -import type { Histogram, Summary } from 'prom-client'; +import type { Gauge, Histogram, Summary } from 'prom-client'; import type { CachedSettings } from '../../../settings/server/CachedSettings'; import type { APIClass } from '../ApiClass'; @@ -11,19 +11,28 @@ export const metricsMiddleware = settings, summary, histogram, + responseSizeHistogram, + activeRequestsGauge, }: { basePathRegex?: RegExp; api: APIClass; settings: CachedSettings; summary: Summary; histogram: Histogram; + responseSizeHistogram: Histogram; + activeRequestsGauge: Gauge; }): MiddlewareHandler => async (c, next) => { const rocketchatRestApiEnd = summary.startTimer(); const rocketchatRestApiHistEnd = histogram.startTimer(); + const methodLabel = { method: c.req.method.toLowerCase() }; + activeRequestsGauge.inc(methodLabel); + await next(); + activeRequestsGauge.dec(methodLabel); + const { method, path, routePath } = c.req; // get rid of the base path (i.e.: /api/v1/) @@ -41,4 +50,9 @@ export const metricsMiddleware = ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), }); rocketchatRestApiHistEnd(histogramLabels); + + const contentLength = parseInt(c.res.headers.get('content-length') || '0', 10); + if (contentLength > 0) { + responseSizeHistogram.observe(histogramLabels, contentLength); + } }; diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index c40ae48ee13ed..82e50928183b3 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -429,7 +429,7 @@ const Api = new WebHookAPI({ Api.router .use(loggerMiddleware(integrationLogger)) .use(tracerSpanMiddleware) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })); + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })); Api.addRoute( ':integrationId/:userId/:token', diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index 9293ac0655f0e..379ff54591d29 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -85,6 +85,17 @@ export const metrics = { labelNames: ['method', 'entrypoint', 'status'], buckets: latencyBuckets, }), + rocketchatRestApiResponseSizeBytes: new client.Histogram({ + name: 'rocketchat_rest_api_response_size_bytes', + help: 'histogram of rocketchat rest api response sizes in bytes', + labelNames: ['method', 'entrypoint', 'status'], + buckets: [100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000], + }), + rocketchatRestApiActiveRequests: new client.Gauge({ + name: 'rocketchat_rest_api_active_requests', + help: 'number of currently active rest api requests', + labelNames: ['method'], + }), meteorSubscriptions: new client.Summary({ name: 'rocketchat_meteor_subscriptions', diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index 0c6460fb31fe9..4b46639160dd0 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -72,7 +72,7 @@ export class AppsRestApi { const logger = new Logger('APPS'); this.api.router .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds })) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) .use(tracerSpanMiddleware); this.addManagementRoutes(); From 5b0c8b3743a35e3b3b847ab014877bb8e7d4fd50 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:36:08 +0000 Subject: [PATCH 09/19] reorder integrations api middleware chain to metrics -> tracer -> logger Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/integrations/server/api/api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index 82e50928183b3..a072f06e867bb 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -427,9 +427,9 @@ const Api = new WebHookAPI({ }); Api.router - .use(loggerMiddleware(integrationLogger)) + .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) .use(tracerSpanMiddleware) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })); + .use(loggerMiddleware(integrationLogger)); Api.addRoute( ':integrationId/:userId/:token', From 25ea7b198e45b8c3ac0f29b60c86fb6d54aeacb7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:04:02 +0000 Subject: [PATCH 10/19] fix metrics.spec.ts: add required histogram, responseSizeHistogram, activeRequestsGauge mocks Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/api.ts | 12 ++- .../api/server/middlewares/metrics.spec.ts | 85 ++++++++++++++----- .../meteor/app/integrations/server/api/api.ts | 12 ++- .../ee/server/apps/communication/rest.ts | 12 ++- 4 files changed, 97 insertions(+), 24 deletions(-) diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index cb95019294545..fea1f8179d6aa 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -102,7 +102,17 @@ settings.watch('API_Enable_Rate_Limiter_Limit_Calls_Default', (value) => export const startRestAPI = () => { (WebApp.rawConnectHandlers as unknown as ReturnType).use( API.api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\/v1\//), + api: API.v1, + settings, + summary: metrics.rocketchatRestApi, + histogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) .use(tracerSpanMiddleware) .use(remoteAddressMiddleware) .use(cors(settings)) diff --git a/apps/meteor/app/api/server/middlewares/metrics.spec.ts b/apps/meteor/app/api/server/middlewares/metrics.spec.ts index 6f53a489c6fb5..3c862aa265771 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.spec.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.spec.ts @@ -25,27 +25,42 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); - api.use(metricsMiddleware({ api: { version: 1 } as any, settings, summary: summary as any })).get( - '/test', - { - response: { - 200: ajv.compile({ - type: 'object', - properties: { - message: { type: 'string' }, + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + + api + .use( + metricsMiddleware({ + api: { version: 1 } as any, + settings, + summary: summary as any, + histogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) + .get( + '/test', + { + response: { + 200: ajv.compile({ + type: 'object', + properties: { + message: { type: 'string' }, + }, + }), + }, + }, + async () => { + return { + statusCode: 200, + body: { + message: 'Metrics test successful', }, - }), + }; }, - }, - async () => { - return { - statusCode: 200, - body: { - message: 'Metrics test successful', - }, - }; - }, - ); + ); app.use(api.router); const response = await request(app).get('/api/test').set('user-agent', 'test'); expect(response.statusCode).toBe(200); @@ -73,8 +88,22 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, summary: summary as any })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\//), + api: { version: 1 } as any, + settings, + summary: summary as any, + histogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) .get( '/test', { @@ -120,8 +149,22 @@ describe('Metrics middleware', () => { const mockEndTimer = jest.fn(); summary.startTimer.mockReturnValue(mockEndTimer); + const histogram = { startTimer: jest.fn().mockReturnValue(jest.fn()) }; + const responseSizeHistogram = { observe: jest.fn() }; + const activeRequestsGauge = { inc: jest.fn(), dec: jest.fn() }; + api - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, summary: summary as any })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\//), + api: { version: 1 } as any, + settings, + summary: summary as any, + histogram: histogram as any, + responseSizeHistogram: responseSizeHistogram as any, + activeRequestsGauge: activeRequestsGauge as any, + }), + ) .get( '/method.call/:id', { diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index a072f06e867bb..de764d4ad0392 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -427,7 +427,17 @@ const Api = new WebHookAPI({ }); Api.router - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/hooks\//), + api: Api, + settings, + summary: metrics.rocketchatRestApi, + histogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) .use(tracerSpanMiddleware) .use(loggerMiddleware(integrationLogger)); diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index 4b46639160dd0..032c3d3404eb9 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -72,7 +72,17 @@ export class AppsRestApi { const logger = new Logger('APPS'); this.api.router .use(loggerMiddleware(logger)) - .use(metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, summary: metrics.rocketchatRestApi, histogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests })) + .use( + metricsMiddleware({ + basePathRegex: new RegExp(/^\/api\/apps\//), + api: this.api, + settings, + summary: metrics.rocketchatRestApi, + histogram: metrics.rocketchatRestApiSeconds, + responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, + activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, + }), + ) .use(tracerSpanMiddleware); this.addManagementRoutes(); From a0b5a065681cc290628ace99859c3c314186cb59 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Mar 2026 23:12:47 +0000 Subject: [PATCH 11/19] rename summary/histogram params to endpointTimeSummary/endpointTimeHistogram in metricsMiddleware Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/api/server/api.ts | 4 ++-- .../app/api/server/middlewares/metrics.spec.ts | 12 ++++++------ apps/meteor/app/api/server/middlewares/metrics.ts | 12 ++++++------ apps/meteor/app/integrations/server/api/api.ts | 4 ++-- apps/meteor/ee/server/apps/communication/rest.ts | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apps/meteor/app/api/server/api.ts b/apps/meteor/app/api/server/api.ts index fea1f8179d6aa..1b5bac10a65df 100644 --- a/apps/meteor/app/api/server/api.ts +++ b/apps/meteor/app/api/server/api.ts @@ -107,8 +107,8 @@ export const startRestAPI = () => { basePathRegex: new RegExp(/^\/api\/v1\//), api: API.v1, settings, - summary: metrics.rocketchatRestApi, - histogram: metrics.rocketchatRestApiSeconds, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, }), diff --git a/apps/meteor/app/api/server/middlewares/metrics.spec.ts b/apps/meteor/app/api/server/middlewares/metrics.spec.ts index 3c862aa265771..3fdc94cda826a 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.spec.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.spec.ts @@ -34,8 +34,8 @@ describe('Metrics middleware', () => { metricsMiddleware({ api: { version: 1 } as any, settings, - summary: summary as any, - histogram: histogram as any, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, responseSizeHistogram: responseSizeHistogram as any, activeRequestsGauge: activeRequestsGauge as any, }), @@ -98,8 +98,8 @@ describe('Metrics middleware', () => { basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, - summary: summary as any, - histogram: histogram as any, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, responseSizeHistogram: responseSizeHistogram as any, activeRequestsGauge: activeRequestsGauge as any, }), @@ -159,8 +159,8 @@ describe('Metrics middleware', () => { basePathRegex: new RegExp(/^\/api\//), api: { version: 1 } as any, settings, - summary: summary as any, - histogram: histogram as any, + endpointTimeSummary: summary as any, + endpointTimeHistogram: histogram as any, responseSizeHistogram: responseSizeHistogram as any, activeRequestsGauge: activeRequestsGauge as any, }), diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index e8423b885b8bb..761f91393976d 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -9,22 +9,22 @@ export const metricsMiddleware = basePathRegex, api, settings, - summary, - histogram, + endpointTimeSummary, + endpointTimeHistogram, responseSizeHistogram, activeRequestsGauge, }: { basePathRegex?: RegExp; api: APIClass; settings: CachedSettings; - summary: Summary; - histogram: Histogram; + endpointTimeSummary: Summary; + endpointTimeHistogram: Histogram; responseSizeHistogram: Histogram; activeRequestsGauge: Gauge; }): MiddlewareHandler => async (c, next) => { - const rocketchatRestApiEnd = summary.startTimer(); - const rocketchatRestApiHistEnd = histogram.startTimer(); + const rocketchatRestApiEnd = endpointTimeSummary.startTimer(); + const rocketchatRestApiHistEnd = endpointTimeHistogram.startTimer(); const methodLabel = { method: c.req.method.toLowerCase() }; activeRequestsGauge.inc(methodLabel); diff --git a/apps/meteor/app/integrations/server/api/api.ts b/apps/meteor/app/integrations/server/api/api.ts index de764d4ad0392..25019e64d3799 100644 --- a/apps/meteor/app/integrations/server/api/api.ts +++ b/apps/meteor/app/integrations/server/api/api.ts @@ -432,8 +432,8 @@ Api.router basePathRegex: new RegExp(/^\/hooks\//), api: Api, settings, - summary: metrics.rocketchatRestApi, - histogram: metrics.rocketchatRestApiSeconds, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, }), diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index 032c3d3404eb9..f01e8bd2447e5 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -77,8 +77,8 @@ export class AppsRestApi { basePathRegex: new RegExp(/^\/api\/apps\//), api: this.api, settings, - summary: metrics.rocketchatRestApi, - histogram: metrics.rocketchatRestApiSeconds, + endpointTimeSummary: metrics.rocketchatRestApi, + endpointTimeHistogram: metrics.rocketchatRestApiSeconds, responseSizeHistogram: metrics.rocketchatRestApiResponseSizeBytes, activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, }), From f9497d71d55c6a555b07c2ddedb3598ba74ab65d Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Thu, 12 Mar 2026 11:35:29 -0300 Subject: [PATCH 12/19] refactor: elu metric name --- apps/meteor/app/metrics/server/lib/metrics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index 379ff54591d29..bc3364b61f74d 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -331,8 +331,8 @@ export const metrics = { let eluBase = performance.eventLoopUtilization(); new client.Gauge({ - name: 'nodejs_event_loop_utilization', - help: 'Event Loop Utilization (ELU) as reported by Node', + name: 'nodejs_event_loop_utilization_ratio', + help: 'Event Loop Utilization (ELU) as reported by NodeJS', collect() { this.set(performance.eventLoopUtilization(eluBase).utilization); eluBase = performance.eventLoopUtilization(); From 7dd09f47b343e31d11f77aa8c9432c1979eb5a24 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:25:12 +0000 Subject: [PATCH 13/19] update bucket values for latency and response size histograms Co-authored-by: d-gubert <1810309+d-gubert@users.noreply.github.com> --- apps/meteor/app/metrics/server/lib/metrics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index bc3364b61f74d..f12f958669dd4 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -4,7 +4,7 @@ import client from 'prom-client'; const percentiles = [0.01, 0.1, 0.5, 0.9, 0.95, 0.99, 1]; -const latencyBuckets = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; +const latencyBuckets = [0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0]; const queueWaitBuckets = [1, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600]; export const metrics = { @@ -89,7 +89,7 @@ export const metrics = { name: 'rocketchat_rest_api_response_size_bytes', help: 'histogram of rocketchat rest api response sizes in bytes', labelNames: ['method', 'entrypoint', 'status'], - buckets: [100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000], + buckets: [0, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000], }), rocketchatRestApiActiveRequests: new client.Gauge({ name: 'rocketchat_rest_api_active_requests', From 6c07a8575d538e9830a5f581f73d693fec1b90a7 Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Thu, 23 Apr 2026 15:29:32 -0300 Subject: [PATCH 14/19] fix: email notification couter incremented twice Co-authored-by: Copilot --- .../app/lib/server/functions/notifications/email.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/meteor/app/lib/server/functions/notifications/email.js b/apps/meteor/app/lib/server/functions/notifications/email.js index 6c92dee3b2bc1..4de543abb7dd6 100644 --- a/apps/meteor/app/lib/server/functions/notifications/email.js +++ b/apps/meteor/app/lib/server/functions/notifications/email.js @@ -178,15 +178,14 @@ export async function getEmailData({ message, receiver, sender, subscription, ro )}${message.tmid || message._id}@${replyto.split('@')[1]}`; } - metrics.notificationsSent.inc({ notification_type: 'email' }); - metrics.notificationsSentTotal.inc({ notification_type: 'email' }); return email; } export function sendEmailFromData(data) { - metrics.notificationsSent.inc({ notification_type: 'email' }); - metrics.notificationsSentTotal.inc({ notification_type: 'email' }); - return Mailer.send(data); + return Mailer.send(data).then(() => { + metrics.notificationsSent.inc({ notification_type: 'email' }); + metrics.notificationsSentTotal.inc({ notification_type: 'email' }); + }); } export function shouldNotifyEmail({ From 03e0635392f2e81b940aec2b5edf4cec790f396d Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Thu, 23 Apr 2026 15:36:06 -0300 Subject: [PATCH 15/19] fix: message sent count Co-authored-by: Copilot --- apps/meteor/app/lib/server/lib/processDirectEmail.ts | 10 ++++++---- apps/meteor/app/lib/server/methods/sendMessage.ts | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/meteor/app/lib/server/lib/processDirectEmail.ts b/apps/meteor/app/lib/server/lib/processDirectEmail.ts index af4a1ebad7fe2..e930add5716e8 100644 --- a/apps/meteor/app/lib/server/lib/processDirectEmail.ts +++ b/apps/meteor/app/lib/server/lib/processDirectEmail.ts @@ -109,9 +109,6 @@ export const processDirectEmail = async function (email: ParsedMail): Promise = { ts: tsDiff < 10000 ? ts : new Date(), msg, @@ -121,5 +118,10 @@ export const processDirectEmail = async function (email: ParsedMail): Promise Date: Thu, 23 Apr 2026 15:38:05 -0300 Subject: [PATCH 16/19] fix: mobile push notification count inc --- .../app/push-notifications/server/lib/PushNotification.ts | 3 ++- apps/meteor/server/services/video-conference/service.ts | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts index 13ef97a198ad9..33c752825ae97 100644 --- a/apps/meteor/app/push-notifications/server/lib/PushNotification.ts +++ b/apps/meteor/app/push-notifications/server/lib/PushNotification.ts @@ -103,9 +103,10 @@ class PushNotification { idOnly, }); + await Push.send(config); + metrics.notificationsSent.inc({ notification_type: 'mobile' }); metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); - await Push.send(config); } async getNotificationForMessageId({ diff --git a/apps/meteor/server/services/video-conference/service.ts b/apps/meteor/server/services/video-conference/service.ts index ac0d287b8fa0b..fc6080d68a6f7 100644 --- a/apps/meteor/server/services/video-conference/service.ts +++ b/apps/meteor/server/services/video-conference/service.ts @@ -674,8 +674,6 @@ export class VideoConfService extends ServiceClassInternal implements IVideoConf return; } - metrics.notificationsSent.inc({ notification_type: 'mobile' }); - metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); await Push.send({ from: 'push', badge: 0, @@ -702,6 +700,9 @@ export class VideoConfService extends ServiceClassInternal implements IVideoConf category: 'VIDEOCONF', }, }); + + metrics.notificationsSent.inc({ notification_type: 'mobile' }); + metrics.notificationsSentTotal.inc({ notification_type: 'mobile' }); } private async sendAllPushNotifications(callId: VideoConference['_id']): Promise { From 0dc40cedacad2730b57258d40433bd3413e11f98 Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Thu, 23 Apr 2026 16:03:28 -0300 Subject: [PATCH 17/19] fix: middleware ordering for /api/apps --- apps/meteor/ee/server/apps/communication/rest.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/meteor/ee/server/apps/communication/rest.ts b/apps/meteor/ee/server/apps/communication/rest.ts index f01e8bd2447e5..24e6e82d6c944 100644 --- a/apps/meteor/ee/server/apps/communication/rest.ts +++ b/apps/meteor/ee/server/apps/communication/rest.ts @@ -70,8 +70,8 @@ export class AppsRestApi { }); const logger = new Logger('APPS'); + this.api.router - .use(loggerMiddleware(logger)) .use( metricsMiddleware({ basePathRegex: new RegExp(/^\/api\/apps\//), @@ -83,7 +83,8 @@ export class AppsRestApi { activeRequestsGauge: metrics.rocketchatRestApiActiveRequests, }), ) - .use(tracerSpanMiddleware); + .use(tracerSpanMiddleware) + .use(loggerMiddleware(logger)); this.addManagementRoutes(); // Using the same instance of the existing API for now, to be able to use the same api prefix(/api) From 48a60ac8fcbe335393a0e2c08ea275de3e61d461 Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Wed, 6 May 2026 17:52:24 -0300 Subject: [PATCH 18/19] refactor: keep version label on new rest api metric --- apps/meteor/app/api/server/middlewares/metrics.ts | 3 ++- apps/meteor/app/metrics/server/lib/metrics.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/meteor/app/api/server/middlewares/metrics.ts b/apps/meteor/app/api/server/middlewares/metrics.ts index 761f91393976d..75774c495dbaf 100644 --- a/apps/meteor/app/api/server/middlewares/metrics.ts +++ b/apps/meteor/app/api/server/middlewares/metrics.ts @@ -41,14 +41,15 @@ export const metricsMiddleware = const histogramLabels = { status: c.res.status, method: method.toLowerCase(), + version: api.version, entrypoint: basePathRegex && entrypoint.startsWith('method.call') ? decodeURIComponent(path.replace(basePathRegex, '')) : entrypoint, }; rocketchatRestApiEnd({ ...histogramLabels, - version: api.version, ...(settings.get('Prometheus_API_User_Agent') && { user_agent: c.req.header('user-agent') }), }); + rocketchatRestApiHistEnd(histogramLabels); const contentLength = parseInt(c.res.headers.get('content-length') || '0', 10); diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index f12f958669dd4..778647c698d5b 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -82,7 +82,7 @@ export const metrics = { rocketchatRestApiSeconds: new client.Histogram({ name: 'rocketchat_rest_api_seconds', help: 'histogram of rocketchat rest api count and time in seconds', - labelNames: ['method', 'entrypoint', 'status'], + labelNames: ['method', 'entrypoint', 'status', 'version'], buckets: latencyBuckets, }), rocketchatRestApiResponseSizeBytes: new client.Histogram({ From 18a844a8416eb18cea3f567ecea35912736fa4ba Mon Sep 17 00:00:00 2001 From: Douglas Gubert Date: Mon, 18 May 2026 15:01:40 -0300 Subject: [PATCH 19/19] fix: api response size metric version label --- apps/meteor/app/metrics/server/lib/metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/meteor/app/metrics/server/lib/metrics.ts b/apps/meteor/app/metrics/server/lib/metrics.ts index 778647c698d5b..939b2edbd0089 100644 --- a/apps/meteor/app/metrics/server/lib/metrics.ts +++ b/apps/meteor/app/metrics/server/lib/metrics.ts @@ -88,7 +88,7 @@ export const metrics = { rocketchatRestApiResponseSizeBytes: new client.Histogram({ name: 'rocketchat_rest_api_response_size_bytes', help: 'histogram of rocketchat rest api response sizes in bytes', - labelNames: ['method', 'entrypoint', 'status'], + labelNames: ['method', 'entrypoint', 'status', 'version'], buckets: [0, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000], }), rocketchatRestApiActiveRequests: new client.Gauge({