@ -3,6 +3,7 @@ import http from 'http';
import client from 'prom-client' ;
import connect from 'connect' ;
import _ from 'underscore' ;
import gcStats from 'prometheus-gc-stats' ;
import { Meteor } from 'meteor/meteor' ;
import { Info , getOplogInfo } from '../../../utils/server' ;
@ -11,45 +12,55 @@ import { settings } from '../../../settings';
import { Statistics } from '../../../models' ;
import { oplogEvents } from '../../../models/server/oplogEvents' ;
client . collectDefaultMetrics ( ) ;
export const metrics = { } ;
const percentiles = [ 0.01 , 0.1 , 0.9 , 0.99 ] ;
// Metrics
metrics . metricsRequests = new client . Counter ( { name : 'rocketchat_metrics_requests' , labelNames : [ 'notification_type' ] , help : 'cumulated number of calls to the metrics endpoint' } ) ;
metrics . metricsSize = new client . Gauge ( { name : 'rocketchat_metrics_size' , help : 'size of the metrics response in chars' } ) ;
metrics . info = new client . Gauge ( { name : 'rocketchat_info' , labelNames : [ 'version' , 'unique_id' , 'site_url' ] , help : 'Rocket.Chat info' } ) ;
metrics . meteorMethods = new client . Summary ( {
name : 'rocketchat_meteor_methods' ,
help : 'summary of meteor methods count and time' ,
labelNames : [ 'method' , 'has_connection' , 'has_user' ] ,
percentiles ,
} ) ;
metrics . rocketchatCallbacks = new client . Summary ( {
name : 'rocketchat_callbacks' ,
help : 'summary of rocketchat callbacks count and time' ,
labelNames : [ 'hook' , 'callback' ] ,
percentiles ,
} ) ;
metrics . rocketchatHooks = new client . Summary ( {
name : 'rocketchat_hooks' ,
help : 'summary of rocketchat hooks count and time' ,
labelNames : [ 'hook' , 'callbacks_length' ] ,
percentiles ,
} ) ;
metrics . rocketchatRestApi = new client . Summary ( {
name : 'rocketchat_rest_api' ,
help : 'summary of rocketchat rest api count and time' ,
labelNames : [ 'method' , 'entrypoint' , 'user_agent' , 'status' , 'version' ] ,
percentiles ,
} ) ;
metrics . meteorSubscriptions = new client . Summary ( {
name : 'rocketchat_meteor_subscriptions' ,
help : 'summary of meteor subscriptions count and time' ,
labelNames : [ 'subscription' ] ,
percentiles ,
} ) ;
metrics . messagesSent = new client . Counter ( { name : 'rocketchat_message_sent' , help : 'cumulated number of messages sent' } ) ;
metrics . notificationsSent = new client . Counter ( { name : 'rocketchat_notification_sent' , labelNames : [ 'notification_type' ] , help : 'cumulated number of notifications sent' } ) ;
metrics . ddpSessions = new client . Gauge ( { name : 'rocketchat_ddp_sessions_count' , help : 'number of open ddp sessions' } ) ;
metrics . ddpAthenticatedSessions = new client . Gauge ( { name : 'rocketchat_ddp_sessions_auth' , help : 'number of authenticated open ddp sessions' } ) ;
metrics . ddpAu thenticatedSessions = new client . Gauge ( { name : 'rocketchat_ddp_sessions_auth' , help : 'number of authenticated open ddp sessions' } ) ;
metrics . ddpConnectedUsers = new client . Gauge ( { name : 'rocketchat_ddp_connected_users' , help : 'number of unique connected users' } ) ;
metrics . ddpRateLimitExceeded = new client . Counter ( { name : 'rocketchat_ddp_rate_limit_exceeded' , labelNames : [ 'limit_name' , 'user_id' , 'client_address' , 'type' , 'name' , 'connection_id' ] , help : 'number of times a ddp rate limiter was exceeded' } ) ;
@ -87,57 +98,52 @@ metrics.totalDirectMessages = new client.Gauge({ name: 'rocketchat_direct_messag
metrics . totalLivechatMessages = new client . Gauge ( { name : 'rocketchat_livechat_messages_total' , help : 'total of messages in livechat rooms' } ) ;
const setPrometheusData = async ( ) => {
client . register . setDefaultLabels ( {
uniqueId : settings . get ( 'uniqueID' ) ,
siteUrl : settings . get ( 'Site_Url' ) ,
} ) ;
const date = new Date ( ) ;
client . register . setDefaultLabels ( {
metrics . info . set ( {
version : Info . version ,
unique _id : settings . get ( 'uniqueID' ) ,
site _url : settings . get ( 'Site_Url' ) ,
version : Info . version ,
} ) ;
} , 1 ) ;
const sessions = Array . from ( Meteor . server . sessions . values ( ) ) ;
const authenticatedSessions = sessions . filter ( ( s ) => s . userId ) ;
metrics . ddpSessions . set ( Meteor . server . sessions . size , date ) ;
metrics . ddpAthenticatedSessions . set ( authenticatedSessions . length , date ) ;
metrics . ddpConnectedUsers . set ( _ . unique ( authenticatedSessions . map ( ( s ) => s . userId ) ) . length , date ) ;
metrics . ddpSessions . set ( Meteor . server . sessions . size ) ;
metrics . ddpAu thenticatedSessions . set ( authenticatedSessions . length ) ;
metrics . ddpConnectedUsers . set ( _ . unique ( authenticatedSessions . map ( ( s ) => s . userId ) ) . length ) ;
const statistics = Statistics . findLast ( ) ;
if ( ! statistics ) {
return ;
}
metrics . version . set ( { version : statistics . version } , 1 , date ) ;
metrics . migration . set ( Migrations . _getControl ( ) . version , date ) ;
metrics . instanceCount . set ( statistics . instanceCount , date ) ;
metrics . oplogEnabled . set ( { enabled : statistics . oplogEnabled } , 1 , date ) ;
metrics . version . set ( { version : statistics . version } , 1 ) ;
metrics . migration . set ( Migrations . _getControl ( ) . version ) ;
metrics . instanceCount . set ( statistics . instanceCount ) ;
metrics . oplogEnabled . set ( { enabled : statistics . oplogEnabled } , 1 ) ;
// User statistics
metrics . totalUsers . set ( statistics . totalUsers , date ) ;
metrics . activeUsers . set ( statistics . activeUsers , date ) ;
metrics . nonActiveUsers . set ( statistics . nonActiveUsers , date ) ;
metrics . onlineUsers . set ( statistics . onlineUsers , date ) ;
metrics . awayUsers . set ( statistics . awayUsers , date ) ;
metrics . offlineUsers . set ( statistics . offlineUsers , date ) ;
metrics . totalUsers . set ( statistics . totalUsers ) ;
metrics . activeUsers . set ( statistics . activeUsers ) ;
metrics . nonActiveUsers . set ( statistics . nonActiveUsers ) ;
metrics . onlineUsers . set ( statistics . onlineUsers ) ;
metrics . awayUsers . set ( statistics . awayUsers ) ;
metrics . offlineUsers . set ( statistics . offlineUsers ) ;
// Room statistics
metrics . totalRooms . set ( statistics . totalRooms , date ) ;
metrics . totalChannels . set ( statistics . totalChannels , date ) ;
metrics . totalPrivateGroups . set ( statistics . totalPrivateGroups , date ) ;
metrics . totalDirect . set ( statistics . totalDirect , date ) ;
metrics . totalLivechat . set ( statistics . totalLivechat , date ) ;
metrics . totalRooms . set ( statistics . totalRooms ) ;
metrics . totalChannels . set ( statistics . totalChannels ) ;
metrics . totalPrivateGroups . set ( statistics . totalPrivateGroups ) ;
metrics . totalDirect . set ( statistics . totalDirect ) ;
metrics . totalLivechat . set ( statistics . totalLivechat ) ;
// Message statistics
metrics . totalMessages . set ( statistics . totalMessages , date ) ;
metrics . totalChannelMessages . set ( statistics . totalChannelMessages , date ) ;
metrics . totalPrivateGroupMessages . set ( statistics . totalPrivateGroupMessages , date ) ;
metrics . totalDirectMessages . set ( statistics . totalDirectMessages , date ) ;
metrics . totalLivechatMessages . set ( statistics . totalLivechatMessages , date ) ;
metrics . totalMessages . set ( statistics . totalMessages ) ;
metrics . totalChannelMessages . set ( statistics . totalChannelMessages ) ;
metrics . totalPrivateGroupMessages . set ( statistics . totalPrivateGroupMessages ) ;
metrics . totalDirectMessages . set ( statistics . totalDirectMessages ) ;
metrics . totalLivechatMessages . set ( statistics . totalLivechatMessages ) ;
const oplogQueue = getOplogInfo ( ) . mongo . _oplogHandle ? . _entryQueue ? . length || 0 ;
metrics . oplogQueue . set ( oplogQueue , date ) ;
metrics . oplogQueue . set ( oplogQueue ) ;
} ;
const app = connect ( ) ;
@ -147,7 +153,12 @@ const app = connect();
app . use ( '/metrics' , ( req , res ) => {
res . setHeader ( 'Content-Type' , 'text/plain' ) ;
res . end ( client . register . metrics ( ) ) ;
const data = client . register . metrics ( ) ;
metrics . metricsRequests . inc ( ) ;
metrics . metricsSize . set ( data . length ) ;
res . end ( data ) ;
} ) ;
app . use ( '/' , ( req , res ) => {
@ -175,35 +186,79 @@ const oplogMetric = ({ collection, op }) => {
} ;
let timer ;
let wasEnabled = false ;
let resetTimer ;
let defaultMetricsInitiated = false ;
let gcStatsInitiated = false ;
const was = {
enabled : false ,
port : 9458 ,
resetInterval : 0 ,
collectGC : false ,
} ;
const updatePrometheusConfig = async ( ) => {
const port = process . env . PROMETHEUS _PORT || settings . get ( 'Prometheus_Port' ) ;
const enabled = Boolean ( port && settings . get ( 'Prometheus_Enabled' ) ) ;
if ( wasEnabled === enabled ) {
const is = {
port : process . env . PROMETHEUS _PORT || settings . get ( 'Prometheus_Port' ) ,
enabled : settings . get ( 'Prometheus_Enabled' ) ,
resetInterval : settings . get ( 'Prometheus_Reset_Interval' ) ,
collectGC : settings . get ( 'Prometheus_Garbage_Collector' ) ,
} ;
if ( Object . values ( is ) . some ( ( s ) => s == null ) ) {
return ;
}
wasEnabled = enabled ;
if ( Object . entries ( is ) . every ( ( [ k , v ] ) => v === was [ k ] ) ) {
return ;
}
if ( ! enabled ) {
server . close ( ) ;
Meteor . clearInterval ( timer ) ;
oplogEvents . removeListener ( 'record' , oplogMetric ) ;
if ( ! is . enabled ) {
if ( was . enabled ) {
console . log ( 'Disabling Prometheus' ) ;
server . close ( ) ;
Meteor . clearInterval ( timer ) ;
oplogEvents . removeListener ( 'record' , oplogMetric ) ;
}
Object . assign ( was , is ) ;
return ;
}
server . listen ( {
port ,
host : process . env . BIND _IP || '0.0.0.0' ,
} ) ;
console . log ( 'Configuring Prometheus' , is ) ;
if ( ! was . enabled ) {
server . listen ( {
port : is . port ,
host : process . env . BIND _IP || '0.0.0.0' ,
} ) ;
timer = Meteor . setInterval ( setPrometheusData , 5000 ) ;
timer = Meteor . setInterval ( setPrometheusData , 5000 ) ;
oplogEvents . on ( 'record' , oplogMetric ) ;
}
Meteor . clearInterval ( resetTimer ) ;
if ( is . resetInterval ) {
resetTimer = Meteor . setInterval ( ( ) => {
client . register . getMetricsAsArray ( ) . forEach ( ( metric ) => { metric . hashMap = { } ; } ) ;
} , is . resetInterval ) ;
}
// Prevent exceptions on calling those methods twice since
// it's not possible to stop them to be able to restart
try {
if ( defaultMetricsInitiated === false ) {
defaultMetricsInitiated = true ;
client . collectDefaultMetrics ( ) ;
}
if ( is . collectGC && gcStatsInitiated === false ) {
gcStatsInitiated = true ;
gcStats ( ) ( ) ;
}
} catch ( error ) {
console . error ( error ) ;
}
oplogEvents . on ( 'record' , oplogMetric ) ;
Object . assign ( was , is ) ;
} ;
Meteor . startup ( async ( ) => {
settings . get ( 'Prometheus_Enabled' , updatePrometheusConfig ) ;
settings . get ( 'Prometheus_Port' , updatePrometheusConfig ) ;
settings . get ( /^Prometheus_.+/ , updatePrometheusConfig ) ;
} ) ;