From db2492c77d0d48b24525bde42eddd5cf495bb62b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 15 May 2018 15:06:20 -0300 Subject: [PATCH] Add setting and expose prometheus on port 9100 (#10766) * Add setting and expose prometheus on port 9100 * Prometheus: Add number of connected users * Send statistics to prometheus * Prometheus: Add methods, subscriptions and callbacks data * Prometheus: Add metrics of REST API calls * Prometheus: Record subscriptions time * Add metrics to notifications --- packages/rocketchat-api/package.js | 1 - packages/rocketchat-api/server/api.js | 14 +- .../rocketchat-api/server/default/metrics.js | 8 - packages/rocketchat-lib/lib/callbacks.js | 5 + .../server/functions/notifications/audio.js | 1 + .../server/functions/notifications/desktop.js | 1 + .../server/functions/notifications/email.js | 1 + .../server/lib/PushNotification.js | 1 + packages/rocketchat-lib/server/lib/debug.js | 12 +- packages/rocketchat-lib/server/lib/metrics.js | 141 ++++++++++++++++++ .../rocketchat-lib/server/startup/settings.js | 9 +- 11 files changed, 181 insertions(+), 13 deletions(-) delete mode 100644 packages/rocketchat-api/server/default/metrics.js diff --git a/packages/rocketchat-api/package.js b/packages/rocketchat-api/package.js index f9704385541..46d6194bdbd 100644 --- a/packages/rocketchat-api/package.js +++ b/packages/rocketchat-api/package.js @@ -27,7 +27,6 @@ Package.onUse(function(api) { //Add default routes api.addFiles('server/default/info.js', 'server'); - api.addFiles('server/default/metrics.js', 'server'); //Add v1 routes api.addFiles('server/v1/channels.js', 'server'); diff --git a/packages/rocketchat-api/server/api.js b/packages/rocketchat-api/server/api.js index 29bd0c08c84..227c21391f3 100644 --- a/packages/rocketchat-api/server/api.js +++ b/packages/rocketchat-api/server/api.js @@ -146,16 +146,26 @@ class API extends Restivus { //Add a try/catch for each endpoint const originalAction = endpoints[method].action; endpoints[method].action = function _internalRouteActionHandler() { + const rocketchatRestApiEnd = RocketChat.metrics.rocketchatRestApi.startTimer({ + method, + entrypoint: route + }); this.logger.debug(`${ this.request.method.toUpperCase() }: ${ this.request.url }`); let result; try { result = originalAction.apply(this); } catch (e) { this.logger.debug(`${ method } ${ route } threw an error:`, e.stack); - return RocketChat.API.v1.failure(e.message, e.error); + result = RocketChat.API.v1.failure(e.message, e.error); } - return result ? result : RocketChat.API.v1.success(); + result = result || RocketChat.API.v1.success(); + + rocketchatRestApiEnd({ + status: result.statusCode + }); + + return result; }; for (const [name, helperMethod] of this.getHelperMethods()) { diff --git a/packages/rocketchat-api/server/default/metrics.js b/packages/rocketchat-api/server/default/metrics.js deleted file mode 100644 index 89b761c199a..00000000000 --- a/packages/rocketchat-api/server/default/metrics.js +++ /dev/null @@ -1,8 +0,0 @@ -RocketChat.API.default.addRoute('metrics', { authRequired: false }, { - get() { - return { - headers: { 'Content-Type': 'text/plain' }, - body: RocketChat.promclient.register.metrics() - }; - } -}); diff --git a/packages/rocketchat-lib/lib/callbacks.js b/packages/rocketchat-lib/lib/callbacks.js index f4f8e70865a..3744491b0a3 100644 --- a/packages/rocketchat-lib/lib/callbacks.js +++ b/packages/rocketchat-lib/lib/callbacks.js @@ -80,6 +80,10 @@ RocketChat.callbacks.run = function(hook, item, constant) { const result = _.sortBy(callbacks, function(callback) { return callback.priority || RocketChat.callbacks.priority.MEDIUM; }).reduce(function(result, callback) { + let rocketchatCallbacksEnd; + if (Meteor.isServer) { + rocketchatCallbacksEnd = RocketChat.metrics.rocketchatCallbacks.startTimer({hook, callback: callback.id}); + } let time = 0; if (RocketChat.callbacks.showTime === true || RocketChat.callbacks.showTotalTime === true) { time = Date.now(); @@ -90,6 +94,7 @@ RocketChat.callbacks.run = function(hook, item, constant) { totalTime += currentTime; if (RocketChat.callbacks.showTime === true) { if (Meteor.isServer) { + rocketchatCallbacksEnd(); RocketChat.statsTracker.timing('callbacks.time', currentTime, [`hook:${ hook }`, `callback:${ callback.id }`]); } else { let stack = callback.stack && typeof callback.stack.split === 'function' && callback.stack.split('\n'); diff --git a/packages/rocketchat-lib/server/functions/notifications/audio.js b/packages/rocketchat-lib/server/functions/notifications/audio.js index b77d4395a53..aac7fc14c2d 100644 --- a/packages/rocketchat-lib/server/functions/notifications/audio.js +++ b/packages/rocketchat-lib/server/functions/notifications/audio.js @@ -24,6 +24,7 @@ export function shouldNotifyAudio({ } export function notifyAudioUser(userId, message, room) { + RocketChat.metrics.audioNotificationsSent.inc(); RocketChat.Notifications.notifyUser(userId, 'audioNotification', { payload: { _id: message._id, diff --git a/packages/rocketchat-lib/server/functions/notifications/desktop.js b/packages/rocketchat-lib/server/functions/notifications/desktop.js index a31dd6c10ed..6f5fe8c1f0b 100644 --- a/packages/rocketchat-lib/server/functions/notifications/desktop.js +++ b/packages/rocketchat-lib/server/functions/notifications/desktop.js @@ -30,6 +30,7 @@ export function notifyDesktopUser({ return; } + RocketChat.metrics.desktopNotificationsSent.inc(); RocketChat.Notifications.notifyUser(userId, 'notification', { title, text, diff --git a/packages/rocketchat-lib/server/functions/notifications/email.js b/packages/rocketchat-lib/server/functions/notifications/email.js index 5d4871cf3b6..21b7fda8383 100644 --- a/packages/rocketchat-lib/server/functions/notifications/email.js +++ b/packages/rocketchat-lib/server/functions/notifications/email.js @@ -137,6 +137,7 @@ export function sendEmail({ message, user, subscription, room, emailAddress, toA } Meteor.defer(() => { + RocketChat.metrics.emailNotificationsSent.inc(); Email.send(email); }); } diff --git a/packages/rocketchat-lib/server/lib/PushNotification.js b/packages/rocketchat-lib/server/lib/PushNotification.js index 15e50c780fe..db3ff9137a0 100644 --- a/packages/rocketchat-lib/server/lib/PushNotification.js +++ b/packages/rocketchat-lib/server/lib/PushNotification.js @@ -47,6 +47,7 @@ class PushNotification { }; } + RocketChat.metrics.mobileNotificationsSent.inc(); return Push.send(config); } } diff --git a/packages/rocketchat-lib/server/lib/debug.js b/packages/rocketchat-lib/server/lib/debug.js index 1a000e44b17..103675dd5fb 100644 --- a/packages/rocketchat-lib/server/lib/debug.js +++ b/packages/rocketchat-lib/server/lib/debug.js @@ -14,10 +14,13 @@ const logger = new Logger('Meteor', { const wrapMethods = function(name, originalHandler, methodsMap) { methodsMap[name] = function() { + const end = RocketChat.metrics.meteorMethods.startTimer({method: name}); const args = name === 'ufsWrite' ? Array.prototype.slice.call(arguments, 1) : arguments; logger.method(name, '-> userId:', Meteor.userId(), ', arguments: ', args); - return originalHandler.apply(this, arguments); + const result = originalHandler.apply(this, arguments); + end(); + return result; }; }; @@ -35,6 +38,13 @@ const originalMeteorPublish = Meteor.publish; Meteor.publish = function(name, func) { return originalMeteorPublish(name, function() { logger.publish(name, '-> userId:', this.userId, ', arguments: ', arguments); + const end = RocketChat.metrics.meteorSubscriptions.startTimer({subscription: name}); + + const originalReady = this.ready; + this.ready = function() { + end(); + return originalReady.apply(this, arguments); + }; return func.apply(this, arguments); }); diff --git a/packages/rocketchat-lib/server/lib/metrics.js b/packages/rocketchat-lib/server/lib/metrics.js index 43cf6590f7b..21dcad0bf99 100644 --- a/packages/rocketchat-lib/server/lib/metrics.js +++ b/packages/rocketchat-lib/server/lib/metrics.js @@ -1,9 +1,150 @@ import client from 'prom-client'; +import connect from 'connect'; +import http from 'http'; +import _ from 'underscore'; RocketChat.promclient = client; +client.collectDefaultMetrics(); RocketChat.metrics = {}; // one sample metrics only - a counter +RocketChat.metrics.meteorMethods = new client.Summary({ + name: 'meteor_methods', + help: 'summary of meteor methods count and time', + labelNames: ['method'] +}); +RocketChat.metrics.meteorMethods.observe(10); + +RocketChat.metrics.rocketchatCallbacks = new client.Summary({ + name: 'rocketchat_callbacks', + help: 'summary of rocketchat callbacks count and time', + labelNames: ['hook', 'callback'] +}); +RocketChat.metrics.meteorMethods.observe(10); + +RocketChat.metrics.rocketchatRestApi = new client.Summary({ + name: 'rocketchat_rest_api', + help: 'summary of rocketchat rest api count and time', + labelNames: ['method', 'entrypoint', 'status'] +}); +RocketChat.metrics.meteorMethods.observe(10); + +RocketChat.metrics.meteorSubscriptions = new client.Summary({ + name: 'meteor_subscriptions', + help: 'summary of meteor subscriptions count and time', + labelNames: ['subscription'] +}); + RocketChat.metrics.messagesSent = new client.Counter({'name': 'message_sent', 'help': 'cumulated number of messages sent'}); +RocketChat.metrics.audioNotificationsSent = new client.Counter({'name': 'audio_sent', 'help': 'cumulated number of audio notifications sent'}); +RocketChat.metrics.desktopNotificationsSent = new client.Counter({'name': 'desktop_sent', 'help': 'cumulated number of desktop notifications sent'}); +RocketChat.metrics.mobileNotificationsSent = new client.Counter({'name': 'mobile_sent', 'help': 'cumulated number of mobile notifications sent'}); +RocketChat.metrics.emailNotificationsSent = new client.Counter({'name': 'email_sent', 'help': 'cumulated number of email notifications sent'}); + +RocketChat.metrics.ddpSessions = new client.Gauge({'name': 'ddp_sessions_count', 'help': 'number of open ddp sessions'}); +RocketChat.metrics.ddpConnectedUsers = new client.Gauge({'name': 'ddp_connected_users', 'help': 'number of connected users'}); + +RocketChat.metrics.version = new client.Gauge({'name': 'version', labelNames: ['version'], 'help': 'Rocket.Chat version'}); +RocketChat.metrics.migration = new client.Gauge({'name': 'migration', 'help': 'migration versoin'}); +RocketChat.metrics.instanceCount = new client.Gauge({'name': 'instance_count', 'help': 'instances running'}); +RocketChat.metrics.oplogEnabled = new client.Gauge({'name': 'oplog_enabled', labelNames: ['enabled'], 'help': 'oplog enabled'}); + +// User statistics +RocketChat.metrics.totalUsers = new client.Gauge({'name': 'users_total', 'help': 'total of users'}); +RocketChat.metrics.activeUsers = new client.Gauge({'name': 'users_active', 'help': 'total of active users'}); +RocketChat.metrics.nonActiveUsers = new client.Gauge({'name': 'users_non_active', 'help': 'total of non active users'}); +RocketChat.metrics.onlineUsers = new client.Gauge({'name': 'users_online', 'help': 'total of users online'}); +RocketChat.metrics.awayUsers = new client.Gauge({'name': 'users_away', 'help': 'total of users away'}); +RocketChat.metrics.offlineUsers = new client.Gauge({'name': 'users_offline', 'help': 'total of users offline'}); + +// Room statistics +RocketChat.metrics.totalRooms = new client.Gauge({'name': 'rooms_total', 'help': 'total of rooms'}); +RocketChat.metrics.totalChannels = new client.Gauge({'name': 'channels_total', 'help': 'total of public rooms/channels'}); +RocketChat.metrics.totalPrivateGroups = new client.Gauge({'name': 'private_groups_total', 'help': 'total of private rooms'}); +RocketChat.metrics.totalDirect = new client.Gauge({'name': 'direct_total', 'help': 'total of direct rooms'}); +RocketChat.metrics.totalLivechat = new client.Gauge({'name': 'livechat_total', 'help': 'total of livechat rooms'}); + +// Message statistics +RocketChat.metrics.totalMessages = new client.Gauge({'name': 'messages_total', 'help': 'total of messages'}); +RocketChat.metrics.totalChannelMessages = new client.Gauge({'name': 'channel_messages_total', 'help': 'total of messages in public rooms'}); +RocketChat.metrics.totalPrivateGroupMessages = new client.Gauge({'name': 'private_group_messages_total', 'help': 'total of messages in private rooms'}); +RocketChat.metrics.totalDirectMessages = new client.Gauge({'name': 'direct_messages_total', 'help': 'total of messages in direct rooms'}); +RocketChat.metrics.totalLivechatMessages = new client.Gauge({'name': 'livechat_messages_total', 'help': 'total of messages in livechat rooms'}); + +client.register.setDefaultLabels({ + uniqueId: RocketChat.settings.get('uniqueID'), + siteUrl: RocketChat.settings.get('Site_Url') +}); + +const setPrometheusData = () => { + const date = new Date(); + + client.register.setDefaultLabels({ + unique_id: RocketChat.settings.get('uniqueID'), + site_url: RocketChat.settings.get('Site_Url'), + version: RocketChat.Info.version + }); + + RocketChat.metrics.ddpSessions.set(Object.keys(Meteor.server.sessions).length, date); + RocketChat.metrics.ddpConnectedUsers.set(_.compact(_.unique(Object.values(Meteor.server.sessions).map(s => s.userId))).length, date); + + const statistics = RocketChat.models.Statistics.findLast(); + if (!statistics) { + return; + } + + RocketChat.metrics.version.set({version: statistics.version}, 1, date); + RocketChat.metrics.migration.set(RocketChat.Migrations._getControl().version, date); + RocketChat.metrics.instanceCount.set(statistics.instanceCount, date); + RocketChat.metrics.oplogEnabled.set({enabled: statistics.oplogEnabled}, 1, date); + + // User statistics + RocketChat.metrics.totalUsers.set(statistics.totalUsers, date); + RocketChat.metrics.activeUsers.set(statistics.activeUsers, date); + RocketChat.metrics.nonActiveUsers.set(statistics.nonActiveUsers, date); + RocketChat.metrics.onlineUsers.set(statistics.onlineUsers, date); + RocketChat.metrics.awayUsers.set(statistics.awayUsers, date); + RocketChat.metrics.offlineUsers.set(statistics.offlineUsers, date); + + // Room statistics + RocketChat.metrics.totalRooms.set(statistics.totalRooms, date); + RocketChat.metrics.totalChannels.set(statistics.totalChannels, date); + RocketChat.metrics.totalPrivateGroups.set(statistics.totalPrivateGroups, date); + RocketChat.metrics.totalDirect.set(statistics.totalDirect, date); + RocketChat.metrics.totalLivechat.set(statistics.totlalLivechat, date); + + // Message statistics + RocketChat.metrics.totalMessages.set(statistics.totalMessages, date); + RocketChat.metrics.totalChannelMessages.set(statistics.totalChannelMessages, date); + RocketChat.metrics.totalPrivateGroupMessages.set(statistics.totalPrivateGroupMessages, date); + RocketChat.metrics.totalDirectMessages.set(statistics.totalDirectMessages, date); + RocketChat.metrics.totalLivechatMessages.set(statistics.totalLivechatMessages, date); +}; + +const app = connect(); + +// const compression = require('compression'); +// app.use(compression()); + +app.use('/metrics', (req, res) => { + res.setHeader('Content-Type', 'text/plain'); + res.end(RocketChat.promclient.register.metrics()); +}); + +const server = http.createServer(app); + +let timer; +RocketChat.settings.get('Prometheus_Enabled', (key, value) => { + if (value === true) { + server.listen({ + port: 9100, + host: process.env.BIND_IP || '0.0.0.0' + }); + timer = Meteor.setInterval(setPrometheusData, 5000); + } else { + server.close(); + Meteor.clearInterval(timer); + } +}); diff --git a/packages/rocketchat-lib/server/startup/settings.js b/packages/rocketchat-lib/server/startup/settings.js index cd857ef925f..cefce97ddea 100644 --- a/packages/rocketchat-lib/server/startup/settings.js +++ b/packages/rocketchat-lib/server/startup/settings.js @@ -1632,9 +1632,16 @@ RocketChat.settings.addGroup('Logs', function() { type: 'boolean', 'public': true }); - return this.add('Log_View_Limit', 1000, { + this.add('Log_View_Limit', 1000, { type: 'int' }); + + this.section('Prometheus', function() { + this.add('Prometheus_Enabled', false, { + type: 'boolean', + i18nLabel: 'Enabled' + }); + }); }); RocketChat.settings.addGroup('Setup_Wizard', function() {