From c349fef66adb6824c6940f91292ccdd772b2fbb3 Mon Sep 17 00:00:00 2001 From: amolghode1981 <86001342+amolghode1981@users.noreply.github.com> Date: Sat, 26 Feb 2022 04:10:24 +0530 Subject: [PATCH] Regression: If Asterisk suddenly goes down, server has no way to know. Causes server to get stuck. Needs restart (#24624) * Clickup Task : https://app.clickup.com/t/22qmrw3 Description: This PR implements a connection retries in case the asterisk connection dies for some reason. It does 5 retries with backoff options. The retries will not be done for connectivity check. Once the retries are exhausted, and server comes up, there is no way to connect back but to restart the voip service. This works only for management connection. Call Server connection, if dies, there are no retries. * Grammar rename on vars Co-authored-by: Kevin Aleman --- .../voip/connector/asterisk/CommandHandler.ts | 18 +- .../connector/asterisk/ami/AMIConnection.ts | 194 ++++++++++++++---- server/services/voip/lib/Helper.ts | 4 + server/services/voip/service.ts | 6 +- 4 files changed, 168 insertions(+), 54 deletions(-) diff --git a/server/services/voip/connector/asterisk/CommandHandler.ts b/server/services/voip/connector/asterisk/CommandHandler.ts index 7c2af2ebe42..0dc3175f3cf 100644 --- a/server/services/voip/connector/asterisk/CommandHandler.ts +++ b/server/services/voip/connector/asterisk/CommandHandler.ts @@ -54,7 +54,7 @@ export class CommandHandler { } if (!config) { this.logger.warn('Management server configuration not found'); - throw Error('Management server configuration not found'); + return; } /** * If we have the same type of connection already established, close it @@ -72,15 +72,13 @@ export class CommandHandler { (config.configData as IManagementConfigData).username, (config.configData as IManagementConfigData).password, ); + this.connections.set(commandType, connection); + this.continuousMonitor = CommandFactory.getCommandObject(Commands.event_stream, this.db); + this.continuousMonitor.connection = this.connections.get(this.continuousMonitor.type) as IConnection; + this.continuousMonitor.initMonitor({}); } catch (error: any) { - this.logger.warn({ msg: 'Management server connection error', error }); - throw Error(`Management server error in connection ${error.message}`); + this.logger.error({ msg: 'Management server connection error', error }); } - - this.connections.set(commandType, connection); - this.continuousMonitor = CommandFactory.getCommandObject(Commands.event_stream, this.db); - this.continuousMonitor.connection = this.connections.get(this.continuousMonitor.type) as IConnection; - this.continuousMonitor.initMonitor({}); } /* Executes |commandToExecute| on a particular command object @@ -94,6 +92,10 @@ export class CommandHandler { executeCommand(commandToExecute: Commands, commandData?: any): Promise { this.logger.debug({ msg: `executeCommand() executing ${Commands[commandToExecute]}` }); const command = CommandFactory.getCommandObject(commandToExecute, this.db); + const connection = this.connections.get(command.type) as IConnection; + if (!connection || !connection.isConnected()) { + throw Error('Connection error'); + } command.connection = this.connections.get(command.type) as IConnection; return command.executeCommand(commandData); } diff --git a/server/services/voip/connector/asterisk/ami/AMIConnection.ts b/server/services/voip/connector/asterisk/ami/AMIConnection.ts index 08c03b6cb24..aa119cd1d30 100644 --- a/server/services/voip/connector/asterisk/ami/AMIConnection.ts +++ b/server/services/voip/connector/asterisk/ami/AMIConnection.ts @@ -31,9 +31,21 @@ function makeLoggerDummy(logger: Logger): Logger { return logger; } +type ConnectionState = 'UNKNOWN' | 'AUTHENTICATED' | 'ERROR'; + export class AMIConnection implements IConnection { connection: typeof Manager | undefined; + connectionState: ConnectionState; + + connectionIpOrHostname: string; + + connectionPort: string; + + userName: string; + + password: string; + eventHandlers: Map; private logger: Logger; @@ -43,10 +55,135 @@ export class AMIConnection implements IConnection { // "Print extended voip connection logs" which will control classes' logging behavior private printLogs = false; + totalReconnectionAttempts = 5; + + currentReconnectionAttempt = 0; + + // Starting with 5 seconds of backoff time. increases with the attempts. + initialBackoffDurationMS = 5000; + + nearEndDisconnect = false; + + // if it is a test connection + // Reconnectivity logic should not be applied. + connectivityCheck = false; + constructor() { const logger = new Logger('AMIConnection'); this.logger = this.printLogs ? logger : makeLoggerDummy(logger); this.eventHandlers = new Map(); + this.connectionState = 'UNKNOWN'; + } + + cleanup(): void { + if (!this.connection) { + return; + } + this.connection.removeAllListeners(); + this.connection = null; + } + + reconnect(): void { + this.logger.debug({ + msg: 'reconnect ()', + initialBackoffDurationMS: this.initialBackoffDurationMS, + currentReconnectionAttempt: this.currentReconnectionAttempt, + }); + if (this.currentReconnectionAttempt === this.totalReconnectionAttempts) { + this.logger.info({ msg: 'reconnect () Not attempting to reconnect' }); + // We have exhausted the reconnection attempts or we have authentication error + // We dont want to retry anymore + this.connectionState = 'ERROR'; + return; + } + const backoffTime = this.initialBackoffDurationMS + this.initialBackoffDurationMS * this.currentReconnectionAttempt; + setTimeout(async () => { + try { + await this.attemptConnection(); + } catch (error: unknown) { + this.logger.error({ msg: 'reconnect () attemptConnection() has thrown error', error }); + } + }, backoffTime); + this.currentReconnectionAttempt += 1; + } + + onManagerError(reject: any, error: unknown): void { + this.logger.error({ msg: 'onManagerError () Connection Error', error }); + this.cleanup(); + this.connectionState = 'ERROR'; + if (this.currentReconnectionAttempt === this.totalReconnectionAttempts) { + this.logger.error({ msg: 'onManagerError () reconnection attempts exhausted. Please check connection settings' }); + reject(error); + } else { + this.reconnect(); + } + } + + onManagerConnect(_resolve: any, _reject: any): void { + this.logger.debug({ msg: 'onManagerConnect () Connection Success' }); + this.connection.login(this.onManagerLogin.bind(this, _resolve, _reject)); + } + + onManagerLogin(resolve: any, reject: any, error: unknown): void { + if (error) { + this.logger.error({ msg: 'onManagerLogin () Authentication Error. Not going to reattempt. Fix the credentaials' }); + // Do not reattempt if we have login failure + this.cleanup(); + reject(error); + } else { + this.connectionState = 'AUTHENTICATED'; + this.currentReconnectionAttempt = 0; + /** + * Note : There is no way to release a handler or cleanup the handlers. + * Handlers are released only when the connection is closed. + * Closing the connection and establishing it again for every command is an overhead. + * To avoid that, we have taken a clean, though a bit complex approach. + * We will register for all the manager event. + * + * Each command will register to AMIConnection to receive the events which it is + * interested in. Once the processing is complete, it will unregister. + * + * Handled in this way will avoid disconnection of the connection to cleanup the + * handlers. + * + * Furthermore, we do not want to initiate this when we are checking + * the connectivity. + */ + if (!this.connectivityCheck) { + this.connection.on('managerevent', this.eventHandlerCallback.bind(this)); + } + this.logger.debug({ msg: 'onManagerLogin () Authentication Success, Connected' }); + resolve(); + } + } + + onManagerClose(hadError: unknown): void { + this.logger.error({ msg: 'onManagerClose ()', hadError }); + this.cleanup(); + if (!this.nearEndDisconnect) { + this.reconnect(); + } + } + + onManagerTimeout(): void { + this.logger.debug({ msg: 'onManagerTimeout ()' }); + this.cleanup(); + } + + async attemptConnection(): Promise { + this.connectionState = 'UNKNOWN'; + this.connection = new Manager(undefined, this.connectionIpOrHostname, this.userName, this.password, true); + + const returnPromise = new Promise((_resolve, _reject) => { + this.connection.on('connect', this.onManagerConnect.bind(this, _resolve, _reject)); + this.connection.on('error', this.onManagerError.bind(this, _reject)); + + this.connection.on('close', this.onManagerClose.bind(this)); + this.connection.on('timeout', this.onManagerTimeout.bind(this)); + + this.connection.connect(this.connectionPort, this.connectionIpOrHostname); + }); + return returnPromise; } /** @@ -81,50 +218,12 @@ export class AMIConnection implements IConnection { connectivityCheck = false, ): Promise { this.logger.log({ msg: 'connect()' }); - this.connection = new Manager(undefined, connectionIpOrHostname, userName, password, true); - const returnPromise = new Promise((_resolve, _reject) => { - const onError = (error: any): void => { - _reject(error); - this.logger.error({ msg: 'connect () Connection Error', error }); - }; - const onConnect = (): void => { - this.logger.debug({ msg: 'connect () Connection Success' }); - }; - const onLogin = (error: any): void => { - if (error) { - _reject(error); - this.logger.error({ msg: 'connect () Authentication error', error }); - } else { - /** - * Note : There is no way to release a handler or cleanup the handlers. - * Handlers are released only when the connection is closed. - * Closing the connection and establishing it again for every command is an overhead. - * To avoid that, we have taken a clean, though a bit complex approach. - * We will register for all the manager event. - * - * Each command will register to AMIConnection to receive the events which it is - * interested in. Once the processing is complete, it will unregister. - * - * Handled in this way will avoid disconnection of the connection to cleanup the - * handlers. - * - * Furthermore, we do not want to initiate this when we are checking - * the connectivity. - */ - if (!connectivityCheck) { - this.connection.on('managerevent', this.eventHandlerCallback.bind(this)); - } - this.logger.debug({ msg: 'connect () Authentication Success, Connected' }); - _resolve(); - } - }; - this.connection.on('connect', onConnect); - this.connection.on('error', onError); - - this.connection.connect(connectionPort, connectionIpOrHostname); - this.connection.login(onLogin); - }); - return returnPromise; + this.connectionIpOrHostname = connectionIpOrHostname; + this.connectionPort = connectionPort; + this.userName = userName; + this.password = password; + this.connectivityCheck = connectivityCheck; + await this.attemptConnection(); } isConnected(): boolean { @@ -136,6 +235,10 @@ export class AMIConnection implements IConnection { // Executes an action on asterisk and returns the result. executeCommand(action: object, actionResultCallback: any): void { + if (this.connectionState !== 'AUTHENTICATED' || (this.connection && !this.connection.isConnected())) { + this.logger.warn({ msg: 'executeCommand() Cant execute command at this moment. Connection is not active' }); + throw Error('Cant execute command at this moment. Connection is not active'); + } this.logger.info({ msg: 'executeCommand()' }); this.connection.action(action, actionResultCallback); } @@ -145,7 +248,6 @@ export class AMIConnection implements IConnection { this.logger.info({ msg: `No event handler set for ${event.event}` }); return; } - const handlers: CallbackContext[] = this.eventHandlers.get(event.event.toLowerCase()); this.logger.debug({ msg: `eventHandlerCallback() Handler count = ${handlers.length}` }); /* Go thru all the available handlers and call each one of them if the actionid matches */ @@ -197,6 +299,8 @@ export class AMIConnection implements IConnection { closeConnection(): void { this.logger.info({ msg: 'closeConnection()' }); + this.nearEndDisconnect = true; this.connection.disconnect(); + this.cleanup(); } } diff --git a/server/services/voip/lib/Helper.ts b/server/services/voip/lib/Helper.ts index 0bae4cf442e..d7a612041c6 100644 --- a/server/services/voip/lib/Helper.ts +++ b/server/services/voip/lib/Helper.ts @@ -31,3 +31,7 @@ export function getServerConfigDataFromSettings(type: ServerType): IVoipCallServ } } } + +export function voipEnabled(): boolean { + return settings.get('VoIP_Enabled'); +} diff --git a/server/services/voip/service.ts b/server/services/voip/service.ts index 8472fe6ec1a..ba298597409 100644 --- a/server/services/voip/service.ts +++ b/server/services/voip/service.ts @@ -16,7 +16,7 @@ import { Commands } from './connector/asterisk/Commands'; import { IVoipConnectorResult } from '../../../definition/IVoipConnectorResult'; import { IQueueMembershipDetails, IRegistrationInfo, isIExtensionDetails } from '../../../definition/IVoipExtension'; import { IQueueDetails, IQueueSummary } from '../../../definition/ACDQueues'; -import { getServerConfigDataFromSettings } from './lib/Helper'; +import { getServerConfigDataFromSettings, voipEnabled } from './lib/Helper'; import { IManagementServerConnectionStatus } from '../../../definition/IVoipServerConnectivityStatus'; export class VoipService extends ServiceClassInternal implements IVoipService { @@ -30,6 +30,10 @@ export class VoipService extends ServiceClassInternal implements IVoipService { super(); this.logger = new Logger('VoIPService'); + if (!voipEnabled()) { + this.logger.warn({ msg: 'Voip is not enabled. Cant start the service' }); + return; + } this.commandHandler = new CommandHandler(db); this.init(); }