Add setting and expose prometheus on port 9100 (#10766)

* Add setting and expose prometheus on port 9100

* Prometheus: Add number of connected users

* Send statistics to prometheus

* Prometheus: Add methods, subscriptions and callbacks data

* Prometheus: Add metrics of REST API calls

* Prometheus: Record subscriptions time

* Add metrics to notifications

packages/rocketchat-api/package.js

@@ -27,7 +27,6 @@ Package.onUse(function(api) {
 
 	//Add default routes
 	api.addFiles('server/default/info.js', 'server');
-	api.addFiles('server/default/metrics.js', 'server');
 
 	//Add v1 routes
 	api.addFiles('server/v1/channels.js', 'server');

packages/rocketchat-api/server/api.js

@@ -146,16 +146,26 @@ class API extends Restivus {
 					//Add a try/catch for each endpoint
 					const originalAction = endpoints[method].action;
 					endpoints[method].action = function _internalRouteActionHandler() {
+						const rocketchatRestApiEnd = RocketChat.metrics.rocketchatRestApi.startTimer({
+							method,
+							entrypoint: route
+						});
 						this.logger.debug(`${ this.request.method.toUpperCase() }: ${ this.request.url }`);
 						let result;
 						try {
 							result = originalAction.apply(this);
 						} catch (e) {
 							this.logger.debug(`${ method } ${ route } threw an error:`, e.stack);
-							return RocketChat.API.v1.failure(e.message, e.error);
+							result = RocketChat.API.v1.failure(e.message, e.error);
 						}
 
-						return result ? result : RocketChat.API.v1.success();
+						result = result || RocketChat.API.v1.success();
+
+						rocketchatRestApiEnd({
+							status: result.statusCode
+						});
+
+						return result;
 					};
 
 					for (const [name, helperMethod] of this.getHelperMethods()) {

packages/rocketchat-api/server/default/metrics.js

@@ -1,8 +0,0 @@
-RocketChat.API.default.addRoute('metrics', { authRequired: false }, {
-	get() {
-		return {
-			headers: { 'Content-Type': 'text/plain' },
-			body: RocketChat.promclient.register.metrics()
-		};
-	}
-});

packages/rocketchat-lib/lib/callbacks.js

@@ -80,6 +80,10 @@ RocketChat.callbacks.run = function(hook, item, constant) {
 		const result = _.sortBy(callbacks, function(callback) {
 			return callback.priority || RocketChat.callbacks.priority.MEDIUM;
 		}).reduce(function(result, callback) {
+			let rocketchatCallbacksEnd;
+			if (Meteor.isServer) {
+				rocketchatCallbacksEnd = RocketChat.metrics.rocketchatCallbacks.startTimer({hook, callback: callback.id});
+			}
 			let time = 0;
 			if (RocketChat.callbacks.showTime === true || RocketChat.callbacks.showTotalTime === true) {
 				time = Date.now();
@@ -90,6 +94,7 @@ RocketChat.callbacks.run = function(hook, item, constant) {
 				totalTime += currentTime;
 				if (RocketChat.callbacks.showTime === true) {
 					if (Meteor.isServer) {
+						rocketchatCallbacksEnd();
 						RocketChat.statsTracker.timing('callbacks.time', currentTime, [`hook:${ hook }`, `callback:${ callback.id }`]);
 					} else {
 						let stack = callback.stack && typeof callback.stack.split === 'function' && callback.stack.split('\n');

packages/rocketchat-lib/server/functions/notifications/audio.js

@@ -24,6 +24,7 @@ export function shouldNotifyAudio({
 }
 
 export function notifyAudioUser(userId, message, room) {
+	RocketChat.metrics.audioNotificationsSent.inc();
 	RocketChat.Notifications.notifyUser(userId, 'audioNotification', {
 		payload: {
 			_id: message._id,

packages/rocketchat-lib/server/functions/notifications/desktop.js

@@ -30,6 +30,7 @@ export function notifyDesktopUser({
 		return;
 	}
 
+	RocketChat.metrics.desktopNotificationsSent.inc();
 	RocketChat.Notifications.notifyUser(userId, 'notification', {
 		title,
 		text,

packages/rocketchat-lib/server/functions/notifications/email.js

@@ -137,6 +137,7 @@ export function sendEmail({ message, user, subscription, room, emailAddress, toA
 	}
 
 	Meteor.defer(() => {
+		RocketChat.metrics.emailNotificationsSent.inc();
 		Email.send(email);
 	});
 }

packages/rocketchat-lib/server/lib/PushNotification.js

@@ -47,6 +47,7 @@ class PushNotification {
 			};
 		}
 
+		RocketChat.metrics.mobileNotificationsSent.inc();
 		return Push.send(config);
 	}
 }

packages/rocketchat-lib/server/lib/debug.js

@@ -14,10 +14,13 @@ const logger = new Logger('Meteor', {
 
 const wrapMethods = function(name, originalHandler, methodsMap) {
 	methodsMap[name] = function() {
+		const end = RocketChat.metrics.meteorMethods.startTimer({method: name});
 		const args = name === 'ufsWrite' ? Array.prototype.slice.call(arguments, 1) : arguments;
 		logger.method(name, '-> userId:', Meteor.userId(), ', arguments: ', args);
 
-		return originalHandler.apply(this, arguments);
+		const result = originalHandler.apply(this, arguments);
+		end();
+		return result;
 	};
 };
 
@@ -35,6 +38,13 @@ const originalMeteorPublish = Meteor.publish;
 Meteor.publish = function(name, func) {
 	return originalMeteorPublish(name, function() {
 		logger.publish(name, '-> userId:', this.userId, ', arguments: ', arguments);
+		const end = RocketChat.metrics.meteorSubscriptions.startTimer({subscription: name});
+
+		const originalReady = this.ready;
+		this.ready = function() {
+			end();
+			return originalReady.apply(this, arguments);
+		};
 
 		return func.apply(this, arguments);
 	});

packages/rocketchat-lib/server/lib/metrics.js

@@ -1,9 +1,150 @@
 import client from 'prom-client';
+import connect from 'connect';
+import http from 'http';
+import _ from 'underscore';
 
 RocketChat.promclient = client;
+client.collectDefaultMetrics();
 
 RocketChat.metrics = {};
 
 // one sample metrics only - a counter
 
+RocketChat.metrics.meteorMethods = new client.Summary({
+	name: 'meteor_methods',
+	help: 'summary of meteor methods count and time',
+	labelNames: ['method']
+});
+RocketChat.metrics.meteorMethods.observe(10);
+
+RocketChat.metrics.rocketchatCallbacks = new client.Summary({
+	name: 'rocketchat_callbacks',
+	help: 'summary of rocketchat callbacks count and time',
+	labelNames: ['hook', 'callback']
+});
+RocketChat.metrics.meteorMethods.observe(10);
+
+RocketChat.metrics.rocketchatRestApi = new client.Summary({
+	name: 'rocketchat_rest_api',
+	help: 'summary of rocketchat rest api count and time',
+	labelNames: ['method', 'entrypoint', 'status']
+});
+RocketChat.metrics.meteorMethods.observe(10);
+
+RocketChat.metrics.meteorSubscriptions = new client.Summary({
+	name: 'meteor_subscriptions',
+	help: 'summary of meteor subscriptions count and time',
+	labelNames: ['subscription']
+});
+
 RocketChat.metrics.messagesSent = new client.Counter({'name': 'message_sent', 'help': 'cumulated number of messages sent'});
+RocketChat.metrics.audioNotificationsSent = new client.Counter({'name': 'audio_sent', 'help': 'cumulated number of audio notifications sent'});
+RocketChat.metrics.desktopNotificationsSent = new client.Counter({'name': 'desktop_sent', 'help': 'cumulated number of desktop notifications sent'});
+RocketChat.metrics.mobileNotificationsSent = new client.Counter({'name': 'mobile_sent', 'help': 'cumulated number of mobile notifications sent'});
+RocketChat.metrics.emailNotificationsSent = new client.Counter({'name': 'email_sent', 'help': 'cumulated number of email notifications sent'});
+
+RocketChat.metrics.ddpSessions = new client.Gauge({'name': 'ddp_sessions_count', 'help': 'number of open ddp sessions'});
+RocketChat.metrics.ddpConnectedUsers = new client.Gauge({'name': 'ddp_connected_users', 'help': 'number of connected users'});
+
+RocketChat.metrics.version = new client.Gauge({'name': 'version', labelNames: ['version'], 'help': 'Rocket.Chat version'});
+RocketChat.metrics.migration = new client.Gauge({'name': 'migration', 'help': 'migration versoin'});
+RocketChat.metrics.instanceCount = new client.Gauge({'name': 'instance_count', 'help': 'instances running'});
+RocketChat.metrics.oplogEnabled = new client.Gauge({'name': 'oplog_enabled', labelNames: ['enabled'], 'help': 'oplog enabled'});
+
+// User statistics
+RocketChat.metrics.totalUsers = new client.Gauge({'name': 'users_total', 'help': 'total of users'});
+RocketChat.metrics.activeUsers = new client.Gauge({'name': 'users_active', 'help': 'total of active users'});
+RocketChat.metrics.nonActiveUsers = new client.Gauge({'name': 'users_non_active', 'help': 'total of non active users'});
+RocketChat.metrics.onlineUsers = new client.Gauge({'name': 'users_online', 'help': 'total of users online'});
+RocketChat.metrics.awayUsers = new client.Gauge({'name': 'users_away', 'help': 'total of users away'});
+RocketChat.metrics.offlineUsers = new client.Gauge({'name': 'users_offline', 'help': 'total of users offline'});
+
+// Room statistics
+RocketChat.metrics.totalRooms = new client.Gauge({'name': 'rooms_total', 'help': 'total of rooms'});
+RocketChat.metrics.totalChannels = new client.Gauge({'name': 'channels_total', 'help': 'total of public rooms/channels'});
+RocketChat.metrics.totalPrivateGroups = new client.Gauge({'name': 'private_groups_total', 'help': 'total of private rooms'});
+RocketChat.metrics.totalDirect = new client.Gauge({'name': 'direct_total', 'help': 'total of direct rooms'});
+RocketChat.metrics.totalLivechat = new client.Gauge({'name': 'livechat_total', 'help': 'total of livechat rooms'});
+
+// Message statistics
+RocketChat.metrics.totalMessages = new client.Gauge({'name': 'messages_total', 'help': 'total of messages'});
+RocketChat.metrics.totalChannelMessages = new client.Gauge({'name': 'channel_messages_total', 'help': 'total of messages in public rooms'});
+RocketChat.metrics.totalPrivateGroupMessages = new client.Gauge({'name': 'private_group_messages_total', 'help': 'total of messages in private rooms'});
+RocketChat.metrics.totalDirectMessages = new client.Gauge({'name': 'direct_messages_total', 'help': 'total of messages in direct rooms'});
+RocketChat.metrics.totalLivechatMessages = new client.Gauge({'name': 'livechat_messages_total', 'help': 'total of messages in livechat rooms'});
+
+client.register.setDefaultLabels({
+	uniqueId: RocketChat.settings.get('uniqueID'),
+	siteUrl: RocketChat.settings.get('Site_Url')
+});
+
+const setPrometheusData = () => {
+	const date = new Date();
+
+	client.register.setDefaultLabels({
+		unique_id: RocketChat.settings.get('uniqueID'),
+		site_url: RocketChat.settings.get('Site_Url'),
+		version: RocketChat.Info.version
+	});
+
+	RocketChat.metrics.ddpSessions.set(Object.keys(Meteor.server.sessions).length, date);
+	RocketChat.metrics.ddpConnectedUsers.set(_.compact(_.unique(Object.values(Meteor.server.sessions).map(s => s.userId))).length, date);
+
+	const statistics = RocketChat.models.Statistics.findLast();
+	if (!statistics) {
+		return;
+	}
+
+	RocketChat.metrics.version.set({version: statistics.version}, 1, date);
+	RocketChat.metrics.migration.set(RocketChat.Migrations._getControl().version, date);
+	RocketChat.metrics.instanceCount.set(statistics.instanceCount, date);
+	RocketChat.metrics.oplogEnabled.set({enabled: statistics.oplogEnabled}, 1, date);
+
+	// User statistics
+	RocketChat.metrics.totalUsers.set(statistics.totalUsers, date);
+	RocketChat.metrics.activeUsers.set(statistics.activeUsers, date);
+	RocketChat.metrics.nonActiveUsers.set(statistics.nonActiveUsers, date);
+	RocketChat.metrics.onlineUsers.set(statistics.onlineUsers, date);
+	RocketChat.metrics.awayUsers.set(statistics.awayUsers, date);
+	RocketChat.metrics.offlineUsers.set(statistics.offlineUsers, date);
+
+	// Room statistics
+	RocketChat.metrics.totalRooms.set(statistics.totalRooms, date);
+	RocketChat.metrics.totalChannels.set(statistics.totalChannels, date);
+	RocketChat.metrics.totalPrivateGroups.set(statistics.totalPrivateGroups, date);
+	RocketChat.metrics.totalDirect.set(statistics.totalDirect, date);
+	RocketChat.metrics.totalLivechat.set(statistics.totlalLivechat, date);
+
+	// Message statistics
+	RocketChat.metrics.totalMessages.set(statistics.totalMessages, date);
+	RocketChat.metrics.totalChannelMessages.set(statistics.totalChannelMessages, date);
+	RocketChat.metrics.totalPrivateGroupMessages.set(statistics.totalPrivateGroupMessages, date);
+	RocketChat.metrics.totalDirectMessages.set(statistics.totalDirectMessages, date);
+	RocketChat.metrics.totalLivechatMessages.set(statistics.totalLivechatMessages, date);
+};
+
+const app = connect();
+
+// const compression = require('compression');
+// app.use(compression());
+
+app.use('/metrics', (req, res) => {
+	res.setHeader('Content-Type', 'text/plain');
+	res.end(RocketChat.promclient.register.metrics());
+});
+
+const server = http.createServer(app);
+
+let timer;
+RocketChat.settings.get('Prometheus_Enabled', (key, value) => {
+	if (value === true) {
+		server.listen({
+			port: 9100,
+			host: process.env.BIND_IP || '0.0.0.0'
+		});
+		timer = Meteor.setInterval(setPrometheusData, 5000);
+	} else {
+		server.close();
+		Meteor.clearInterval(timer);
+	}
+});

packages/rocketchat-lib/server/startup/settings.js

@@ -1632,9 +1632,16 @@ RocketChat.settings.addGroup('Logs', function() {
 		type: 'boolean',
 		'public': true
 	});
-	return this.add('Log_View_Limit', 1000, {
+	this.add('Log_View_Limit', 1000, {
 		type: 'int'
 	});
+
+	this.section('Prometheus', function() {
+		this.add('Prometheus_Enabled', false, {
+			type: 'boolean',
+			i18nLabel: 'Enabled'
+		});
+	});
 });
 
 RocketChat.settings.addGroup('Setup_Wizard', function() {