[NEW] Limit all DDP/Websocket requests (configurable via admin panel) (#13311)

packages/rocketchat-api/server/settings.js

@@ -4,9 +4,6 @@ RocketChat.settings.addGroup('General', function() {
 	this.section('REST API', function() {
 		this.add('API_Upper_Count_Limit', 100, { type: 'int', public: false });
 		this.add('API_Default_Count', 50, { type: 'int', public: false });
-		this.add('API_Enable_Rate_Limiter_Dev', true, { type: 'boolean', public: false });
-		this.add('API_Enable_Rate_Limiter_Limit_Calls_Default', 10, { type: 'int', public: false });
-		this.add('API_Enable_Rate_Limiter_Limit_Time_Default', 60000, { type: 'int', public: false });
 		this.add('API_Allow_Infinite_Count', true, { type: 'boolean', public: false });
 		this.add('API_Enable_Direct_Message_History_EndPoint', false, { type: 'boolean', public: false });
 		this.add('API_Enable_Shields', true, { type: 'boolean', public: false });

packages/rocketchat-i18n/i18n/en.i18n.json

@@ -948,6 +948,21 @@
   "days": "days",
   "DB_Migration": "Database Migration",
   "DB_Migration_Date": "Database Migration Date",
+  "DDP_Rate_Limit_IP_Enabled": "Limit by IP: enabled",
+  "DDP_Rate_Limit_IP_Requests_Allowed": "Limit by IP: requests allowed",
+  "DDP_Rate_Limit_IP_Interval_Time": "Limit by IP: interval time",
+  "DDP_Rate_Limit_User_Enabled": "Limit by User: enabled",
+  "DDP_Rate_Limit_User_Requests_Allowed": "Limit by User: requests allowed",
+  "DDP_Rate_Limit_User_Interval_Time": "Limit by User: interval time",
+  "DDP_Rate_Limit_Connection_Enabled": "Limit by Connection: enabled",
+  "DDP_Rate_Limit_Connection_Requests_Allowed": "Limit by Connection: requests allowed",
+  "DDP_Rate_Limit_Connection_Interval_Time": "Limit by Connection: interval time",
+  "DDP_Rate_Limit_User_By_Method_Enabled": "Limit by User per Method: enabled",
+  "DDP_Rate_Limit_User_By_Method_Requests_Allowed": "Limit by User per Method: requests allowed",
+  "DDP_Rate_Limit_User_By_Method_Interval_Time": "Limit by User per Method: interval time",
+  "DDP_Rate_Limit_Connection_By_Method_Enabled": "Limit by Connection per Method: enabled",
+  "DDP_Rate_Limit_Connection_By_Method_Requests_Allowed": "Limit by Connection per Method: requests allowed",
+  "DDP_Rate_Limit_Connection_By_Method_Interval_Time": "Limit by Connection per Method: interval time",
   "Deactivate": "Deactivate",
   "Decline": "Decline",
   "Decode_Key": "Decode Key",

packages/rocketchat-lib/package.js

@@ -272,6 +272,8 @@ Package.onUse(function(api) {
 	api.addFiles('startup/defaultRoomTypes.js');
 	api.addFiles('startup/index.js', 'server');
 
+	api.addFiles('server/startup/rateLimiter.js', 'server');
+
 	// EXPORT
 	api.export('RocketChat');
 	api.export('handleError', 'client');

packages/rocketchat-lib/server/startup/rateLimiter.js

@@ -0,0 +1,187 @@
+import _ from 'underscore';
+import { Meteor } from 'meteor/meteor';
+import { DDPRateLimiter } from 'meteor/ddp-rate-limiter';
+import { RateLimiter } from 'meteor/rate-limit';
+import { settings } from 'meteor/rocketchat:settings';
+import { metrics } from 'meteor/rocketchat:metrics';
+
+// Get initial set of names already registered for rules
+const names = new Set(Object.values(DDPRateLimiter.printRules())
+	.map((rule) => rule._matchers)
+	.filter((match) => typeof match.name === 'string')
+	.map((match) => match.name));
+
+// Override the addRule to save new names added after this point
+const { addRule } = DDPRateLimiter;
+DDPRateLimiter.addRule = (matcher, calls, time, callback) => {
+	if (matcher && typeof matcher.name === 'string') {
+		names.add(matcher.name);
+	}
+	return addRule.call(DDPRateLimiter, matcher, calls, time, callback);
+};
+
+// Need to override the meteor's code duo to a problem with the callback reply
+// being shared among all matchs
+RateLimiter.prototype.check = function(input) {
+	const self = this;
+	const reply = {
+		allowed: true,
+		timeToReset: 0,
+		numInvocationsLeft: Infinity,
+	};
+
+	const matchedRules = self._findAllMatchingRules(input);
+	_.each(matchedRules, function(rule) {
+		// ==== BEGIN OVERRIDE ====
+		const callbackReply = {
+			allowed: true,
+			timeToReset: 0,
+			numInvocationsLeft: Infinity,
+		};
+		// ==== END OVERRIDE ====
+
+		const ruleResult = rule.apply(input);
+		let numInvocations = rule.counters[ruleResult.key];
+
+		if (ruleResult.timeToNextReset < 0) {
+			// Reset all the counters since the rule has reset
+			rule.resetCounter();
+			ruleResult.timeSinceLastReset = new Date().getTime() - rule._lastResetTime;
+			ruleResult.timeToNextReset = rule.options.intervalTime;
+			numInvocations = 0;
+		}
+
+		if (numInvocations > rule.options.numRequestsAllowed) {
+			// Only update timeToReset if the new time would be longer than the
+			// previously set time. This is to ensure that if this input triggers
+			// multiple rules, we return the longest period of time until they can
+			// successfully make another call
+			if (reply.timeToReset < ruleResult.timeToNextReset) {
+				reply.timeToReset = ruleResult.timeToNextReset;
+			}
+			reply.allowed = false;
+			reply.numInvocationsLeft = 0;
+
+			// ==== BEGIN OVERRIDE ====
+			callbackReply.timeToReset = ruleResult.timeToNextReset;
+			callbackReply.allowed = false;
+			callbackReply.numInvocationsLeft = 0;
+			rule._executeCallback(callbackReply, input);
+			// ==== END OVERRIDE ====
+		} else {
+			// If this is an allowed attempt and we haven't failed on any of the
+			// other rules that match, update the reply field.
+			if (rule.options.numRequestsAllowed - numInvocations < reply.numInvocationsLeft && reply.allowed) {
+				reply.timeToReset = ruleResult.timeToNextReset;
+				reply.numInvocationsLeft = rule.options.numRequestsAllowed - numInvocations;
+			}
+
+			// ==== BEGIN OVERRIDE ====
+			callbackReply.timeToReset = ruleResult.timeToNextReset;
+			callbackReply.numInvocationsLeft = rule.options.numRequestsAllowed - numInvocations;
+			rule._executeCallback(callbackReply, input);
+			// ==== END OVERRIDE ====
+		}
+	});
+	return reply;
+};
+
+const checkNameNonStream = (name) => name && !names.has(name) && !name.startsWith('stream-');
+const checkNameForStream = (name) => name && !names.has(name) && name.startsWith('stream-');
+
+const ruleIds = {};
+
+const callback = (message, name) => (reply, input) => {
+	if (reply.allowed === false) {
+		console.warn('DDP RATE LIMIT:', message);
+		console.warn(JSON.stringify({ ...reply, ...input }, null, 2));
+		metrics.ddpRateLimitExceeded.inc({
+			limit_name: name,
+			user_id: input.userId,
+			client_address: input.clientAddress,
+			type: input.type,
+			name: input.name,
+			connection_id: input.connectionId,
+		});
+	// } else {
+	// 	console.log('DDP RATE LIMIT:', message);
+	// 	console.log(JSON.stringify({ ...reply, ...input }, null, 2));
+	}
+};
+
+const messages = {
+	IP: 'address',
+	User: 'userId',
+	Connection: 'connectionId',
+	User_By_Method: 'userId per method',
+	Connection_By_Method: 'connectionId per method',
+};
+
+const reconfigureLimit = Meteor.bindEnvironment((name, rules, factor = 1) => {
+	if (ruleIds[name + factor]) {
+		DDPRateLimiter.removeRule(ruleIds[name + factor]);
+	}
+
+	if (!settings.get(`DDP_Rate_Limit_${ name }_Enabled`)) {
+		return;
+	}
+
+	ruleIds[name + factor] = addRule(
+		rules,
+		settings.get(`DDP_Rate_Limit_${ name }_Requests_Allowed`) * factor,
+		settings.get(`DDP_Rate_Limit_${ name }_Interval_Time`) * factor,
+		callback(`limit by ${ messages[name] }`, name)
+	);
+});
+
+const configIP = _.debounce(() => {
+	reconfigureLimit('IP', {
+		clientAddress: (clientAddress) => clientAddress !== '127.0.0.1',
+	});
+}, 1000);
+
+const configUser = _.debounce(() => {
+	reconfigureLimit('User', {
+		userId: (userId) => userId != null,
+	});
+}, 1000);
+
+const configConnection = _.debounce(() => {
+	reconfigureLimit('Connection', {
+		connectionId: () => true,
+	});
+}, 1000);
+
+const configUserByMethod = _.debounce(() => {
+	reconfigureLimit('User_By_Method', {
+		type: () => true,
+		name: checkNameNonStream,
+		userId: (userId) => userId != null,
+	});
+	reconfigureLimit('User_By_Method', {
+		type: () => true,
+		name: checkNameForStream,
+		userId: (userId) => userId != null,
+	}, 4);
+}, 1000);
+
+const configConnectionByMethod = _.debounce(() => {
+	reconfigureLimit('Connection_By_Method', {
+		type: () => true,
+		name: checkNameNonStream,
+		connectionId: () => true,
+	});
+	reconfigureLimit('Connection_By_Method', {
+		type: () => true,
+		name: checkNameForStream,
+		connectionId: () => true,
+	}, 4);
+}, 1000);
+
+if (!process.env.TEST_MODE) {
+	settings.get(/^DDP_Rate_Limit_IP_.+/, configIP);
+	settings.get(/^DDP_Rate_Limit_User_[^B].+/, configUser);
+	settings.get(/^DDP_Rate_Limit_Connection_[^B].+/, configConnection);
+	settings.get(/^DDP_Rate_Limit_User_By_Method_.+/, configUserByMethod);
+	settings.get(/^DDP_Rate_Limit_Connection_By_Method_.+/, configConnectionByMethod);
+}

packages/rocketchat-lib/server/startup/settings.js

@@ -2666,4 +2666,34 @@ RocketChat.settings.addGroup('Setup_Wizard', function() {
 	});
 });
 
+RocketChat.settings.addGroup('Rate Limiter', function() {
+	this.section('DDP Rate Limiter', function() {
+		this.add('DDP_Rate_Limit_IP_Enabled', true, { type: 'boolean' });
+		this.add('DDP_Rate_Limit_IP_Requests_Allowed', 120000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_IP_Enabled', value: true } });
+		this.add('DDP_Rate_Limit_IP_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_IP_Enabled', value: true } });
+
+		this.add('DDP_Rate_Limit_User_Enabled', true, { type: 'boolean' });
+		this.add('DDP_Rate_Limit_User_Requests_Allowed', 1200, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_Enabled', value: true } });
+		this.add('DDP_Rate_Limit_User_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_Enabled', value: true } });
+
+		this.add('DDP_Rate_Limit_Connection_Enabled', true, { type: 'boolean' });
+		this.add('DDP_Rate_Limit_Connection_Requests_Allowed', 600, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_Enabled', value: true } });
+		this.add('DDP_Rate_Limit_Connection_Interval_Time', 60000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_Enabled', value: true } });
+
+		this.add('DDP_Rate_Limit_User_By_Method_Enabled', true, { type: 'boolean' });
+		this.add('DDP_Rate_Limit_User_By_Method_Requests_Allowed', 20, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_By_Method_Enabled', value: true } });
+		this.add('DDP_Rate_Limit_User_By_Method_Interval_Time', 10000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_User_By_Method_Enabled', value: true } });
+
+		this.add('DDP_Rate_Limit_Connection_By_Method_Enabled', true, { type: 'boolean' });
+		this.add('DDP_Rate_Limit_Connection_By_Method_Requests_Allowed', 10, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_By_Method_Enabled', value: true } });
+		this.add('DDP_Rate_Limit_Connection_By_Method_Interval_Time', 10000, { type: 'int', enableQuery: { _id: 'DDP_Rate_Limit_Connection_By_Method_Enabled', value: true } });
+	});
+
+	this.section('API Rate Limiter', function() {
+		this.add('API_Enable_Rate_Limiter_Dev', true, { type: 'boolean' });
+		this.add('API_Enable_Rate_Limiter_Limit_Calls_Default', 10, { type: 'int' });
+		this.add('API_Enable_Rate_Limiter_Limit_Time_Default', 60000, { type: 'int' });
+	});
+});
+
 RocketChat.settings.init();

packages/rocketchat-metrics/server/lib/metrics.js

@@ -48,6 +48,7 @@ metrics.notificationsSent = new client.Counter({ name: 'rocketchat_notification_
 metrics.ddpSessions = new client.Gauge({ name: 'rocketchat_ddp_sessions_count', help: 'number of open ddp sessions' });
 metrics.ddpAthenticatedSessions = new client.Gauge({ name: 'rocketchat_ddp_sessions_auth', help: 'number of authenticated open ddp sessions' });
 metrics.ddpConnectedUsers = new client.Gauge({ name: 'rocketchat_ddp_connected_users', help: 'number of unique connected users' });
+metrics.ddpRateLimitExceeded = new client.Counter({ name: 'rocketchat_ddp_rate_limit_exceeded', labelNames: ['limit_name', 'user_id', 'client_address', 'type', 'name', 'connection_id'], help: 'number of times a ddp rate limiter was exceeded' });
 
 metrics.version = new client.Gauge({ name: 'rocketchat_version', labelNames: ['version'], help: 'Rocket.Chat version' });
 metrics.migration = new client.Gauge({ name: 'rocketchat_migration', help: 'migration versoin' });