[FIX] Performance issues when running Omnichannel job queue dispatcher (#23661)

* Fix Omnichannel job queue dispatcher.

* Keep settings validation.

* Remove console.log.

* Review requests.

* Fix default setting value.

app/livechat/server/lib/Helper.js

@@ -40,6 +40,7 @@ export const createLivechatRoom = (rid, name, guest, roomInfo = {}, extraData =
 
 	const extraRoomInfo = callbacks.run('livechat.beforeRoom', roomInfo, extraData);
 	const { _id, username, token, department: departmentId, status = 'online' } = guest;
+	const newRoomAt = new Date();
 
 	logger.debug(`Creating livechat room for visitor ${ _id }`);
 
@@ -47,10 +48,10 @@ export const createLivechatRoom = (rid, name, guest, roomInfo = {}, extraData =
 		_id: rid,
 		msgs: 0,
 		usersCount: 1,
-		lm: new Date(),
+		lm: newRoomAt,
 		fname: name,
 		t: 'l',
-		ts: new Date(),
+		ts: newRoomAt,
 		departmentId,
 		v: {
 			_id,
@@ -67,6 +68,7 @@ export const createLivechatRoom = (rid, name, guest, roomInfo = {}, extraData =
 			type: OmnichannelSourceType.OTHER,
 			alias: 'unknown',
 		},
+		queuedAt: newRoomAt,
 	}, extraRoomInfo);
 
 	const roomId = Rooms.insert(room);

app/livechat/server/lib/QueueManager.js

@@ -7,7 +7,7 @@ import { callbacks } from '../../../callbacks/server';
 import { Logger } from '../../../logger';
 import { RoutingManager } from './RoutingManager';
 
-const logger = new Logger('QueueMananger');
+const logger = new Logger('QueueManager');
 
 export const saveQueueInquiry = (inquiry) => {
 	LivechatInquiry.queueInquiry(inquiry._id);

app/models/server/models/LivechatInquiry.js

@@ -48,7 +48,7 @@ export class LivechatInquiry extends Base {
 		this.update({
 			_id: inquiryId,
 		}, {
-			$set: { status: 'taken' },
+			$set: { status: 'taken', takenAt: new Date() },
 			$unset: { defaultAgent: 1, estimatedInactivityCloseTimeAt: 1 },
 		});
 	}
@@ -71,9 +71,17 @@ export class LivechatInquiry extends Base {
 		return this.update({
 			_id: inquiryId,
 		}, {
-			$set: {
-				status: 'queued',
-			},
+			$set: { status: 'queued', queuedAt: new Date() },
+			$unset: { takenAt: 1 },
+		});
+	}
+
+	queueInquiryAndRemoveDefaultAgent(inquiryId) {
+		return this.update({
+			_id: inquiryId,
+		}, {
+			$set: { status: 'queued', queuedAt: new Date() },
+			$unset: { takenAt: 1, defaultAgent: 1 },
 		});
 	}
 

app/models/server/models/LivechatRooms.js

@@ -20,6 +20,7 @@ export class LivechatRooms extends Base {
 		this.tryEnsureIndex({ 'v.token': 1 }, { sparse: true });
 		this.tryEnsureIndex({ 'v.token': 1, 'email.thread': 1 }, { sparse: true });
 		this.tryEnsureIndex({ 'v._id': 1 }, { sparse: true });
+		this.tryEnsureIndex({ t: 1, departmentId: 1, closedAt: 1 }, { partialFilterExpression: { closedAt: { $exists: true } } });
 	}
 
 	findLivechat(filter = {}, offset = 0, limit = 20) {
@@ -717,9 +718,8 @@ export class LivechatRooms extends Base {
 			t: 'l',
 		};
 		const update = {
-			$unset: {
-				servedBy: 1,
-			},
+			$set: { queuedAt: new Date() },
+			$unset: { servedBy: 1 },
 		};
 
 		this.update(query, update);

app/models/server/raw/Rooms.js

@@ -27,10 +27,8 @@ export class RoomsRaw extends BaseRaw {
 			{
 				$match: {
 					t: 'l',
-					closedAt: { $exists: true },
-					metrics: { $exists: true },
-					'metrics.chatDuration': { $exists: true },
 					...department && { departmentId: department },
+					closedAt: { $exists: true },
 				},
 			},
 			{ $sort: { closedAt: -1 } },

app/ui-sidenav/client/roomList.js

@@ -172,6 +172,7 @@ const mergeSubRoom = (subscription) => {
 			livechatData: 1,
 			departmentId: 1,
 			source: 1,
+			queuedAt: 1,
 		},
 	};
 
@@ -212,6 +213,7 @@ const mergeSubRoom = (subscription) => {
 		departmentId,
 		ts,
 		source,
+		queuedAt,
 	} = room;
 
 	subscription.lm = subscription.lr ? new Date(Math.max(subscription.lr, lastRoomUpdate)) : lastRoomUpdate;
@@ -249,6 +251,7 @@ const mergeSubRoom = (subscription) => {
 		departmentId,
 		ts,
 		source,
+		queuedAt,
 	});
 };
 
@@ -291,6 +294,7 @@ const mergeRoomSub = (room) => {
 		departmentId,
 		ts,
 		source,
+		queuedAt,
 	} = room;
 
 	Subscriptions.update({
@@ -328,6 +332,7 @@ const mergeRoomSub = (room) => {
 			jitsiTimeout,
 			ts,
 			source,
+			queuedAt,
 			...getLowerCaseNames(room, sub.name, sub.fname),
 		},
 	});

client/views/omnichannel/directory/chats/contextualBar/ChatInfo.js

@@ -50,6 +50,7 @@ function ChatInfo({ id, route }) {
 		priorityId,
 		livechatData,
 		source,
+		queuedAt,
 	} = room || { room: { v: {} } };
 
 	const routePath = useRoute(route || 'omnichannel-directory');
@@ -58,6 +59,7 @@ function ChatInfo({ id, route }) {
 	const hasGlobalEditRoomPermission = hasPermission('save-others-livechat-room-info');
 	const hasLocalEditRoomPermission = servedBy?._id === Meteor.userId();
 	const visitorId = v?._id;
+	const queueStartedAt = queuedAt || ts;
 
 	const dispatchToastMessage = useToastMessageDispatch();
 	useEffect(() => {
@@ -126,13 +128,13 @@ function ChatInfo({ id, route }) {
 							<Info>{topic}</Info>
 						</Field>
 					)}
-					{ts && (
+					{queueStartedAt && (
 						<Field>
 							<Label>{t('Queue_Time')}</Label>
 							{servedBy ? (
-								<Info>{moment(servedBy.ts).from(moment(ts), true)}</Info>
+								<Info>{moment(servedBy.ts).from(moment(queueStartedAt), true)}</Info>
 							) : (
-								<Info>{moment(ts).fromNow(true)}</Info>
+								<Info>{moment(queueStartedAt).fromNow(true)}</Info>
 							)}
 						</Field>
 					)}

client/views/omnichannel/directory/chats/contextualBar/ChatInfoDirectory.js

@@ -45,6 +45,7 @@ function ChatInfoDirectory({ id, route, room }) {
 		responseBy,
 		priorityId,
 		livechatData,
+		queuedAt,
 	} = room || { room: { v: {} } };
 
 	const routePath = useRoute(route || 'omnichannel-directory');
@@ -53,6 +54,7 @@ function ChatInfoDirectory({ id, route, room }) {
 	const hasGlobalEditRoomPermission = hasPermission('save-others-livechat-room-info');
 	const hasLocalEditRoomPermission = servedBy?._id === Meteor.userId();
 	const visitorId = v?._id;
+	const queueStartedAt = queuedAt || ts;
 
 	const dispatchToastMessage = useToastMessageDispatch();
 	useEffect(() => {
@@ -120,13 +122,13 @@ function ChatInfoDirectory({ id, route, room }) {
 							<Info>{topic}</Info>
 						</Field>
 					)}
-					{ts && (
+					{queueStartedAt && (
 						<Field>
 							<Label>{t('Queue_Time')}</Label>
 							{servedBy ? (
-								<Info>{moment(servedBy.ts).from(moment(ts), true)}</Info>
+								<Info>{moment(servedBy.ts).from(moment(queueStartedAt), true)}</Info>
 							) : (
-								<Info>{moment(ts).fromNow(true)}</Info>
+								<Info>{moment(queueStartedAt).fromNow(true)}</Info>
 							)}
 						</Field>
 					)}

definition/IRoom.ts

@@ -120,6 +120,7 @@ export interface IOmnichannelRoom extends Omit<IRoom, 'default' | 'featured' | '
 	responseBy: any;
 	priorityId: any;
 	livechatData: any;
+	queuedAt?: Date;
 }
 
 export const isOmnichannelRoom = (room: IRoom): room is IOmnichannelRoom & IRoom => room.t === 'l';

ee/app/livechat-enterprise/server/hooks/afterTakeInquiry.js

@@ -17,6 +17,6 @@ callbacks.add('livechat.afterTakeInquiry', async (inquiry) => {
 	const { department } = inquiry;
 	debouncedDispatchWaitingQueueStatus(department);
 
-	cbLogger.debug(`Statuses for queue ${ department || 'Public' } updated succesfully`);
+	cbLogger.debug(`Statuses for queue ${ department || 'Public' } updated successfully`);
 	return inquiry;
 }, callbacks.priority.MEDIUM, 'livechat-after-take-inquiry');

ee/app/livechat-enterprise/server/hooks/beforeRoutingChat.js

@@ -31,11 +31,13 @@ callbacks.add('livechat.beforeRouteChat', async (inquiry, agent) => {
 
 	saveQueueInquiry(inquiry);
 
-	const [inq] = await LivechatInquiry.getCurrentSortedQueueAsync({ _id, department });
-	if (inq) {
-		dispatchInquiryPosition(inq);
+	if (settings.get('Omnichannel_calculate_dispatch_service_queue_statistics')) {
+		const [inq] = await LivechatInquiry.getCurrentSortedQueueAsync({ _id, department });
+		if (inq) {
+			dispatchInquiryPosition(inq);
+			cbLogger.debug(`Callback success. Inquiry ${ _id } position has been notified`);
+		}
 	}
 
-	cbLogger.debug(`Callback success. Inquiry ${ _id } position has been notified`);
 	return LivechatInquiry.findOneById(_id);
 }, callbacks.priority.HIGH, 'livechat-before-routing-chat');

ee/app/livechat-enterprise/server/hooks/onAgentAssignmentFailed.ts

@@ -16,8 +16,7 @@ const handleOnAgentAssignmentFailed = async ({ inquiry, room, options }: { inqui
 		const { _id: roomId } = room;
 
 		const { _id: inquiryId } = inquiry;
-		LivechatInquiry.queueInquiry(inquiryId);
-		LivechatInquiry.removeDefaultAgentById(inquiryId);
+		LivechatInquiry.queueInquiryAndRemoveDefaultAgent(inquiryId);
 		LivechatRooms.removeAgentByRoomId(roomId);
 		Subscriptions.removeByRoomId(roomId);
 		dispatchAgentDelegated(roomId, null);

ee/app/livechat-enterprise/server/hooks/onCloseLivechat.js

@@ -1,7 +1,6 @@
 import { callbacks } from '../../../../../app/callbacks';
 import { settings } from '../../../../../app/settings/server';
 import { debouncedDispatchWaitingQueueStatus } from '../lib/Helper';
-import { RoutingManager } from '../../../../../app/livechat/server/lib/RoutingManager';
 import { LivechatEnterprise } from '../lib/LivechatEnterprise';
 
 const onCloseLivechat = (room) => {
@@ -12,10 +11,7 @@ const onCloseLivechat = (room) => {
 	}
 
 	const { departmentId } = room || {};
-	if (!RoutingManager.getConfig().autoAssignAgent) {
-		debouncedDispatchWaitingQueueStatus(departmentId);
-		return room;
-	}
+	debouncedDispatchWaitingQueueStatus(departmentId);
 
 	return room;
 };

ee/app/livechat-enterprise/server/lib/Helper.js

@@ -95,8 +95,17 @@ export const dispatchInquiryPosition = async (inquiry, queueInfo) => {
 };
 
 export const dispatchWaitingQueueStatus = async (department) => {
+	if (!settings.get('Livechat_waiting_queue') && !settings.get('Omnichannel_calculate_dispatch_service_queue_statistics')) {
+		return;
+	}
+
 	helperLogger.debug(`Updating statuses for queue ${ department || 'Public' }`);
 	const queue = await LivechatInquiry.getCurrentSortedQueueAsync({ department });
+
+	if (!queue.length) {
+		return;
+	}
+
 	const queueInfo = await getQueueInfo(department);
 	queue.forEach((inquiry) => {
 		dispatchInquiryPosition(inquiry, queueInfo);
@@ -126,14 +135,11 @@ export const processWaitingQueue = async (department) => {
 
 	if (room && room.servedBy) {
 		const { _id: rid, servedBy: { _id: agentId } } = room;
-		helperLogger.debug(`Inquiry ${ inquiry._id } taken succesfully by agent ${ agentId }. Notifying`);
+		helperLogger.debug(`Inquiry ${ inquiry._id } taken successfully by agent ${ agentId }. Notifying`);
 		return setTimeout(() => {
 			propagateAgentDelegated(rid, agentId);
 		}, 1000);
 	}
-
-	const { departmentId } = room || {};
-	await debouncedDispatchWaitingQueueStatus(departmentId);
 };
 
 export const setPredictedVisitorAbandonmentTime = (room) => {
@@ -254,6 +260,10 @@ export const getLivechatQueueInfo = async (room) => {
 		return null;
 	}
 
+	if (!settings.get('Omnichannel_calculate_dispatch_service_queue_statistics')) {
+		return null;
+	}
+
 	const { _id: rid, departmentId: department } = room;
 	const inquiry = LivechatInquiry.findOneByRoomId(rid, { fields: { _id: 1, status: 1 } });
 	if (!inquiry) {

ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js

@@ -199,7 +199,8 @@ export const LivechatEnterprise = {
 	},
 };
 
-const RACE_TIMEOUT = 1000;
+const DEFAULT_RACE_TIMEOUT = 5000;
+let queueDelayTimeout = DEFAULT_RACE_TIMEOUT;
 
 const queueWorker = {
 	running: false,
@@ -242,9 +243,9 @@ const queueWorker = {
 		}
 
 		const queue = await this.nextQueue();
-		queueLogger.debug(`Executing queue ${ queue || 'Public' } with timeout of ${ RACE_TIMEOUT }`);
+		queueLogger.debug(`Executing queue ${ queue || 'Public' } with timeout of ${ queueDelayTimeout }`);
 
-		setTimeout(this.checkQueue.bind(this, queue), RACE_TIMEOUT);
+		setTimeout(this.checkQueue.bind(this, queue), queueDelayTimeout);
 	},
 
 	async checkQueue(queue) {
@@ -292,3 +293,7 @@ settings.watch('Livechat_enabled', (enabled) => {
 	omnichannelIsEnabled = enabled;
 	omnichannelIsEnabled && RoutingManager.isMethodSet() ? shouldQueueStart() : queueWorker.stop();
 });
+
+settings.watch('Omnichannel_queue_delay_timeout', (timeout) => {
+	queueDelayTimeout = timeout < 1 ? DEFAULT_RACE_TIMEOUT : timeout * 1000;
+});

ee/app/livechat-enterprise/server/settings.ts

@@ -113,6 +113,37 @@ export const createSettings = (): void => {
 				],
 			});
 
+			this.add('Omnichannel_calculate_dispatch_service_queue_statistics', true, {
+				type: 'boolean',
+				group: 'Omnichannel',
+				section: 'Queue_management',
+				i18nLabel: 'Omnichannel_calculate_dispatch_service_queue_statistics',
+				enableQuery: [{ _id: 'Livechat_waiting_queue', value: true }, omnichannelEnabledQuery],
+				enterprise: true,
+				invalidValue: false,
+				modules: [
+					'livechat-enterprise',
+				],
+			});
+
+			this.add('Omnichannel_queue_delay_timeout', 5, {
+				type: 'int',
+				group: 'Omnichannel',
+				section: 'Queue_management',
+				i18nLabel: 'Queue_delay_timeout',
+				i18nDescription: 'Time_in_seconds',
+				enableQuery: [
+					{ _id: 'Livechat_waiting_queue', value: true },
+					{ _id: 'Livechat_Routing_Method', value: { $ne: 'Manual_Selection' } },
+					omnichannelEnabledQuery,
+				],
+				enterprise: true,
+				invalidValue: 5,
+				modules: [
+					'livechat-enterprise',
+				],
+			});
+
 			this.add('Livechat_number_most_recent_chats_estimate_wait_time', 100, {
 				type: 'int',
 				group: 'Omnichannel',

packages/rocketchat-i18n/i18n/en.i18n.json

@@ -3175,6 +3175,8 @@
   "Omnichannel": "Omnichannel",
   "Omnichannel_Directory": "Omnichannel Directory",
   "Omnichannel_appearance": "Omnichannel Appearance",
+  "Omnichannel_calculate_dispatch_service_queue_statistics": "Calculate and dispatch Omnichannel waiting queue statistics",
+  "Omnichannel_calculate_dispatch_service_queue_statistics_Description": "Processing and dispatching waiting queue statistics such as position and estimated waiting time. If *Livechat channel* is not in use, it is recommended to disable this setting and prevent the server from doing unnecessary processes.",
   "Omnichannel_Contact_Center": "Omnichannel Contact Center",
   "Omnichannel_contact_manager_routing": "Assign new conversations to the contact manager",
   "Omnichannel_contact_manager_routing_Description": "This setting allocates a chat to the assigned Contact Manager, as long as the Contact Manager is online when the chat starts",
@@ -3401,6 +3403,7 @@
   "Query_description": "Additional conditions for determining which users to send the email to. Unsubscribed users are automatically removed from the query. It must be a valid JSON. Example: \"{\"createdAt\":{\"$gt\":{\"$date\": \"2015-01-01T00:00:00.000Z\"}}}\"",
   "Query_is_not_valid_JSON": "Query is not valid JSON",
   "Queue": "Queue",
+  "Queue_delay_timeout": "Queue processing delay timeout",
   "Queue_Time": "Queue Time",
   "Queue_management": "Queue Management",
   "quote": "quote",
@@ -4726,4 +4729,4 @@
   "Your_temporary_password_is_password": "Your temporary password is <strong>[password]</strong>.",
   "Your_TOTP_has_been_reset": "Your Two Factor TOTP has been reset.",
   "Your_workspace_is_ready": "Your workspace is ready to use 🎉"
-}
+}

packages/rocketchat-i18n/i18n/pt-BR.i18n.json

@@ -3168,6 +3168,8 @@
   "Omnichannel": "Omnichannel",
   "Omnichannel_Directory": "Diretório Omnichannel",
   "Omnichannel_appearance": "Aparência do Omnichannel",
+  "Omnichannel_calculate_dispatch_service_queue_statistics": "Calcular e propagar dados estatísticos da fila de espera Omnichannel",
+  "Omnichannel_calculate_dispatch_service_queue_statistics_Description": "Proessamento de dados estatísticos da fila de espera Omnichannel como as posição e tempo estimado de espera na fila. Se o *canal Livechat* não estiver em uso, é recomendável desabilitar essa configuração e evitar que o servidor execute processos desnecessários.",
   "Omnichannel_Contact_Center": "Central de Contatos Omnichannel",
   "Omnichannel_contact_manager_routing": "Atribuir novas conversas para o Gerente do Contato",
   "Omnichannel_contact_manager_routing_Description": "Aloca novos chats para o Gerente do Contato(quando atribuído), desde que o Gerente do Contato esteja online quando a conversa é iniciado",
@@ -3394,6 +3396,7 @@
   "Query_description": "Condições adicionais para determinar para quais usuários enviar e-mail. Usuários não inscritos são automaticamente removidos a partir da consulta. Deve ser um JSON válido. Exemplo: `{\"createdAt\": {\"$gt\": {\"$date\": \"2015-01-01T00: 00: 00.000Z\"}}}`",
   "Query_is_not_valid_JSON": "Query não é um JSON válido",
   "Queue": "Fila",
+  "Queue_delay_timeout": "Tempo limite de atraso para processamento da fila",
   "Queue_Time": "Tempo na fila",
   "Queue_management": "Gerenciamento de Fila",
   "quote": "citação",
@@ -4714,4 +4717,4 @@
   "Your_temporary_password_is_password": "Sua senha temporária é <strong>[password]</strong>.",
   "Your_TOTP_has_been_reset": "Seu TOTP de dois fatores foi redefinido.",
   "Your_workspace_is_ready": "O seu espaço de trabalho está pronto para usar 🎉"
-}
+}

server/modules/watchers/publishFields.ts

@@ -100,6 +100,7 @@ export const roomFields = {
 	metrics: 1,
 	ts: 1,
 	waitingResponse: 1,
+	queuedAt: 1,
 
 	// fields used by DMs
 	usernames: 1,