feat!: Implement Cloud communication reliability (#32856)

.changeset/plenty-hairs-camp.md

@@ -0,0 +1,9 @@
+---
+"@rocket.chat/meteor": major
+"@rocket.chat/core-typings": major
+"@rocket.chat/model-typings": major
+"@rocket.chat/models": major
+---
+
+Adds a new collection to store all the workspace cloud tokens to defer the race condition management to MongoDB instead of having to handle it within the settings cache.
+Removes the Cloud_Workspace_Access_Token & Cloud_Workspace_Access_Token_Expires_At settings since they are not going to be used anymore.

apps/meteor/app/api/server/v1/misc.ts

@@ -1,7 +1,7 @@
 import crypto from 'crypto';
 
 import { isOAuthUser, type IUser } from '@rocket.chat/core-typings';
-import { Settings, Users } from '@rocket.chat/models';
+import { Settings, Users, WorkspaceCredentials } from '@rocket.chat/models';
 import {
 	isShieldSvgProps,
 	isSpotlightProps,
@@ -664,6 +664,7 @@ API.v1.addRoute(
 			const settingsIds: string[] = [];
 
 			if (this.bodyParams.setDeploymentAs === 'new-workspace') {
+				await WorkspaceCredentials.unsetCredentialByScope();
 				settingsIds.push(
 					'Cloud_Service_Agree_PrivacyTerms',
 					'Cloud_Workspace_Id',
@@ -675,9 +676,7 @@ API.v1.addRoute(
 					'Cloud_Workspace_PublicKey',
 					'Cloud_Workspace_License',
 					'Cloud_Workspace_Had_Trial',
-					'Cloud_Workspace_Access_Token',
 					'uniqueID',
-					'Cloud_Workspace_Access_Token_Expires_At',
 				);
 			}
 

apps/meteor/app/cloud/server/functions/getWorkspaceAccessToken.ts

@@ -1,15 +1,21 @@
-import { Settings } from '@rocket.chat/models';
+import type { IWorkspaceCredentials } from '@rocket.chat/core-typings';
+import { WorkspaceCredentials } from '@rocket.chat/models';
 
-import { notifyOnSettingChangedById } from '../../../lib/server/lib/notifyListener';
-import { settings } from '../../../settings/server';
 import { getWorkspaceAccessTokenWithScope } from './getWorkspaceAccessTokenWithScope';
 import { retrieveRegistrationStatus } from './retrieveRegistrationStatus';
 
+const hasWorkspaceAccessTokenExpired = (credentials: IWorkspaceCredentials): boolean => new Date() >= credentials.expirationDate;
+
 /**
- * @param {boolean} forceNew
- * @param {string} scope
- * @param {boolean} save
- * @returns string
+ * Returns the access token for the workspace, if it is expired or forceNew is true, it will get a new one
+ * and save it to the database, therefore if this function does not throw an error, it will always return a valid token.
+ *
+ * @param {boolean} forceNew - If true, it will get a new token even if the current one is not expired
+ * @param {string} scope - The scope of the token to get
+ * @param {boolean} save - If true, it will save the new token to the database
+ * @throws {CloudWorkspaceAccessTokenError} If the workspace is not registered (no credentials in the database)
+ *
+ * @returns string - A valid access token for the workspace
  */
 export async function getWorkspaceAccessToken(forceNew = false, scope = '', save = true, throwOnError = false): Promise<string> {
 	const { workspaceRegistered } = await retrieveRegistrationStatus();
@@ -18,26 +24,23 @@ export async function getWorkspaceAccessToken(forceNew = false, scope = '', save
 		return '';
 	}
 
-	const expires = await Settings.findOneById('Cloud_Workspace_Access_Token_Expires_At');
-
-	if (expires === null) {
-		throw new Error('Cloud_Workspace_Access_Token_Expires_At is not set');
+	const workspaceCredentials = await WorkspaceCredentials.getCredentialByScope(scope);
+	if (!workspaceCredentials) {
+		throw new CloudWorkspaceAccessTokenError();
 	}
 
-	const now = new Date();
-
-	if (expires.value && now < expires.value && !forceNew) {
-		return settings.get<string>('Cloud_Workspace_Access_Token');
+	if (!hasWorkspaceAccessTokenExpired(workspaceCredentials) && !forceNew) {
+		return workspaceCredentials.accessToken;
 	}
 
 	const accessToken = await getWorkspaceAccessTokenWithScope(scope, throwOnError);
 
 	if (save) {
-		(await Settings.updateValueById('Cloud_Workspace_Access_Token', accessToken.token)).modifiedCount &&
-			void notifyOnSettingChangedById('Cloud_Workspace_Access_Token');
-
-		(await Settings.updateValueById('Cloud_Workspace_Access_Token_Expires_At', accessToken.expiresAt)).modifiedCount &&
-			void notifyOnSettingChangedById('Cloud_Workspace_Access_Token_Expires_At');
+		await WorkspaceCredentials.updateCredentialByScope({
+			scope,
+			accessToken: accessToken.token,
+			expirationDate: accessToken.expiresAt,
+		});
 	}
 
 	return accessToken.token;

apps/meteor/app/cloud/server/functions/removeWorkspaceRegistrationInfo.ts

@@ -1,4 +1,4 @@
-import { Settings } from '@rocket.chat/models';
+import { Settings, WorkspaceCredentials } from '@rocket.chat/models';
 
 import { notifyOnSettingChangedById } from '../../../lib/server/lib/notifyListener';
 import { retrieveRegistrationStatus } from './retrieveRegistrationStatus';
@@ -9,6 +9,8 @@ export async function removeWorkspaceRegistrationInfo() {
 		return true;
 	}
 
+	await WorkspaceCredentials.removeAllCredentials();
+
 	const settingsIds = [
 		'Cloud_Workspace_Id',
 		'Cloud_Workspace_Name',

apps/meteor/app/cloud/server/functions/saveRegistrationData.ts

@@ -1,10 +1,22 @@
 import { applyLicense } from '@rocket.chat/license';
-import { Settings } from '@rocket.chat/models';
+import { Settings, WorkspaceCredentials } from '@rocket.chat/models';
 
 import { notifyOnSettingChangedById } from '../../../lib/server/lib/notifyListener';
 import { settings } from '../../../settings/server';
 import { syncCloudData } from './syncWorkspace/syncCloudData';
 
+type SaveRegistrationDataDTO = {
+	workspaceId: string;
+	client_name: string;
+	client_id: string;
+	client_secret: string;
+	client_secret_expires_at: number;
+	publicKey: string;
+	registration_client_uri: string;
+};
+
+type ManualSaveRegistrationDataDTO = SaveRegistrationDataDTO & { licenseData: { license: string } };
+
 export async function saveRegistrationData({
 	workspaceId,
 	client_name,
@@ -13,15 +25,7 @@ export async function saveRegistrationData({
 	client_secret_expires_at,
 	publicKey,
 	registration_client_uri,
-}: {
-	workspaceId: string;
-	client_name: string;
-	client_id: string;
-	client_secret: string;
-	client_secret_expires_at: number;
-	publicKey: string;
-	registration_client_uri: string;
-}) {
+}: SaveRegistrationDataDTO) {
 	await saveRegistrationDataBase({
 		workspaceId,
 		client_name,
@@ -43,15 +47,7 @@ async function saveRegistrationDataBase({
 	client_secret_expires_at,
 	publicKey,
 	registration_client_uri,
-}: {
-	workspaceId: string;
-	client_name: string;
-	client_id: string;
-	client_secret: string;
-	client_secret_expires_at: number;
-	publicKey: string;
-	registration_client_uri: string;
-}) {
+}: SaveRegistrationDataDTO) {
 	const settingsData = [
 		{ _id: 'Register_Server', value: true },
 		{ _id: 'Cloud_Workspace_Id', value: workspaceId },
@@ -63,7 +59,13 @@ async function saveRegistrationDataBase({
 		{ _id: 'Cloud_Workspace_Registration_Client_Uri', value: registration_client_uri },
 	];
 
-	const promises = settingsData.map(({ _id, value }) => Settings.updateValueById(_id, value));
+	await WorkspaceCredentials.updateCredentialByScope({
+		scope: '',
+		accessToken: '',
+		expirationDate: new Date(0),
+	});
+
+	const promises = [...settingsData.map(({ _id, value }) => Settings.updateValueById(_id, value))];
 
 	(await Promise.all(promises)).forEach((value, index) => {
 		if (value?.modifiedCount) {
@@ -104,18 +106,7 @@ export async function saveRegistrationDataManual({
 	publicKey,
 	registration_client_uri,
 	licenseData,
-}: {
-	workspaceId: string;
-	client_name: string;
-	client_id: string;
-	client_secret: string;
-	client_secret_expires_at: number;
-	publicKey: string;
-	registration_client_uri: string;
-	licenseData: {
-		license: string;
-	};
-}) {
+}: ManualSaveRegistrationDataDTO) {
 	await saveRegistrationDataBase({
 		workspaceId,
 		client_name,

apps/meteor/ee/server/models/WorkspaceCredentials.ts

@@ -0,0 +1,6 @@
+import { registerModel } from '@rocket.chat/models';
+
+import { db } from '../../../server/database/utils';
+import { WorkspaceCredentialsRaw } from './raw/WorkspaceCredentials';
+
+registerModel('IWorkspaceCredentialsModel', new WorkspaceCredentialsRaw(db));

apps/meteor/ee/server/models/raw/WorkspaceCredentials.ts

@@ -0,0 +1,68 @@
+import type { IWorkspaceCredentials } from '@rocket.chat/core-typings';
+import type { IWorkspaceCredentialsModel } from '@rocket.chat/model-typings';
+import type { Db, DeleteResult, Filter, IndexDescription, UpdateResult } from 'mongodb';
+
+import { BaseRaw } from '../../../../server/models/raw/BaseRaw';
+
+export class WorkspaceCredentialsRaw extends BaseRaw<IWorkspaceCredentials> implements IWorkspaceCredentialsModel {
+	constructor(db: Db) {
+		super(db, 'workspace_credentials');
+	}
+
+	protected modelIndexes(): IndexDescription[] {
+		return [{ key: { scopes: 1, expirationDate: 1, accessToken: 1 }, unique: true }];
+	}
+
+	getCredentialByScope(scope = ''): Promise<IWorkspaceCredentials | null> {
+		const query: Filter<IWorkspaceCredentials> = {
+			scopes: {
+				$all: [scope],
+				$size: 1,
+			},
+		};
+
+		return this.findOne(query);
+	}
+
+	unsetCredentialByScope(scope = ''): Promise<DeleteResult> {
+		const query: Filter<IWorkspaceCredentials> = {
+			scopes: {
+				$all: [scope],
+				$size: 1,
+			},
+		};
+
+		return this.deleteOne(query);
+	}
+
+	updateCredentialByScope({
+		scope,
+		accessToken,
+		expirationDate,
+	}: {
+		scope: string;
+		accessToken: string;
+		expirationDate: Date;
+	}): Promise<UpdateResult> {
+		const record = {
+			$set: {
+				scopes: [scope],
+				accessToken,
+				expirationDate,
+			},
+		};
+
+		const query: Filter<IWorkspaceCredentials> = {
+			scopes: {
+				$all: [scope],
+				$size: 1,
+			},
+		};
+
+		return this.updateOne(query, record, { upsert: true });
+	}
+
+	removeAllCredentials(): Promise<DeleteResult> {
+		return this.col.deleteMany({});
+	}
+}

apps/meteor/ee/server/models/startup.ts

@@ -7,6 +7,7 @@ import('./LivechatPriority');
 import('./OmnichannelServiceLevelAgreements');
 import('./AuditLog');
 import('./ReadReceipts');
+import('./WorkspaceCredentials');
 
 void License.onLicense('livechat-enterprise', () => {
 	import('./CannedResponse');

apps/meteor/server/settings/setup-wizard.ts

@@ -1322,28 +1322,6 @@ export const createSetupWSettings = () =>
 				secret: true,
 			});
 
-			await this.add('Cloud_Workspace_Access_Token', '', {
-				type: 'string',
-				hidden: true,
-				readonly: true,
-				enableQuery: {
-					_id: 'Register_Server',
-					value: true,
-				},
-				secret: true,
-			});
-
-			await this.add('Cloud_Workspace_Access_Token_Expires_At', new Date(0), {
-				type: 'date',
-				hidden: true,
-				readonly: true,
-				enableQuery: {
-					_id: 'Register_Server',
-					value: true,
-				},
-				secret: true,
-			});
-
 			await this.add('Cloud_Workspace_Registration_State', '', {
 				type: 'string',
 				hidden: true,

apps/meteor/server/startup/migrations/index.ts

@@ -48,5 +48,6 @@ import './v312';
 import './v313';
 import './v314';
 import './v315';
+import './v316';
 
 export * from './xrun';

apps/meteor/server/startup/migrations/v316.ts

@@ -0,0 +1,31 @@
+import { Settings, WorkspaceCredentials } from '@rocket.chat/models';
+
+import { addMigration } from '../../lib/migrations';
+
+addMigration({
+	version: 316,
+	name: 'Remove Cloud_Workspace_Access_Token and Cloud_Workspace_Access_Token_Expires_At from the settings collection and add to the WorkspaceCredentials collection',
+	async up() {
+		const workspaceCredentials = await WorkspaceCredentials.getCredentialByScope();
+		if (workspaceCredentials) {
+			return;
+		}
+
+		const accessToken = ((await Settings.getValueById('Cloud_Workspace_Access_Token')) as string) || '';
+		const expirationDate = ((await Settings.getValueById('Cloud_Workspace_Access_Token_Expires_At')) as Date) || new Date(0);
+
+		if (accessToken) {
+			await Settings.removeById('Cloud_Workspace_Access_Token');
+		}
+
+		if (expirationDate) {
+			await Settings.removeById('Cloud_Workspace_Access_Token_Expires_At');
+		}
+
+		await WorkspaceCredentials.updateCredentialByScope({
+			scope: '',
+			accessToken,
+			expirationDate,
+		});
+	},
+});

packages/core-typings/src/ee/IWorkspaceCredentials.ts

@@ -0,0 +1,8 @@
+import type { IRocketChatRecord } from '../IRocketChatRecord';
+
+export interface IWorkspaceCredentials extends IRocketChatRecord {
+	_id: string;
+	scopes: string[];
+	expirationDate: Date;
+	accessToken: string;
+}

packages/core-typings/src/index.ts

@@ -42,6 +42,7 @@ export * from './IUserStatus';
 export * from './IUser';
 
 export * from './ee/IAuditLog';
+export * from './ee/IWorkspaceCredentials';
 
 export * from './import';
 export * from './IIncomingMessage';

packages/model-typings/src/index.ts

@@ -81,3 +81,4 @@ export * from './models/ICronHistoryModel';
 export * from './models/IMigrationsModel';
 export * from './models/IModerationReportsModel';
 export * from './updater';
+export * from './models/IWorkspaceCredentialsModel';

packages/model-typings/src/models/IWorkspaceCredentialsModel.ts

@@ -0,0 +1,11 @@
+import type { IWorkspaceCredentials } from '@rocket.chat/core-typings';
+import type { DeleteResult, UpdateResult } from 'mongodb';
+
+import type { IBaseModel } from './IBaseModel';
+
+export interface IWorkspaceCredentialsModel extends IBaseModel<IWorkspaceCredentials> {
+	getCredentialByScope(scope?: string): Promise<IWorkspaceCredentials | null>;
+	unsetCredentialByScope(scope?: string): Promise<DeleteResult>;
+	updateCredentialByScope(credentials: { scope: string; accessToken: string; expirationDate: Date }): Promise<UpdateResult>;
+	removeAllCredentials(): Promise<DeleteResult>;
+}

packages/models/src/index.ts

@@ -79,6 +79,7 @@ import type {
 	ICronHistoryModel,
 	IMigrationsModel,
 	IModerationReportsModel,
+	IWorkspaceCredentialsModel,
 } from '@rocket.chat/model-typings';
 
 import { proxify } from './proxify';
@@ -173,3 +174,4 @@ export const AuditLog = proxify<IAuditLogModel>('IAuditLogModel');
 export const CronHistory = proxify<ICronHistoryModel>('ICronHistoryModel');
 export const Migrations = proxify<IMigrationsModel>('IMigrationsModel');
 export const ModerationReports = proxify<IModerationReportsModel>('IModerationReportsModel');
+export const WorkspaceCredentials = proxify<IWorkspaceCredentialsModel>('IWorkspaceCredentialsModel');