[IMPROVE][APPS] New storage strategy for Apps-Engine file packages (#22657)

* Adjustments and started adding GridFS storage

* Add GridFS support

* Error handling

* Finish GridFS storage

* Update FS storage to latest Apps-Engine changes

* Remove dangling files from gridfs after app update

* Enable the user to choose a storage for the apps' source code (#22717)

* Enable the user to choose a storage for the apps' source code

* Rename app storage proxy module

* Adjust operation of the app source storage proxy module

* Make setting fs path for app storage its own function

* Remove log statement

* [NEW] Migration: apps new storage strategy (#22857)

* Migrate apps source file to gridfs

* Remove unused dependencies

* Convert base64 zip to buffer

* Prevent AppServerOrchestrator from initializing twice

* Refactor migration to use existing models instead of custom code

* Add description for noop on promise catch

Co-authored-by: Douglas Gubert <douglas.gubert@gmail.com>

* Remove unsued properties from apps documents

* Adjustment based on linter feedback

* Rename migration file's extension

* Update apps-engine version

* Adjustments based on linter feedback

Co-authored-by: thassiov <tvmcarvalho@gmail.com>

app/apps/server/orchestrator.js

@@ -12,18 +12,25 @@ import { AppMessagesConverter, AppRoomsConverter, AppSettingsConverter, AppUsers
 import { AppDepartmentsConverter } from './converters/departments';
 import { AppUploadsConverter } from './converters/uploads';
 import { AppVisitorsConverter } from './converters/visitors';
-import { AppRealLogsStorage, AppRealStorage } from './storage';
+import { AppRealLogsStorage, AppRealStorage, ConfigurableAppSourceStorage } from './storage';
 
 function isTesting() {
 	return process.env.TEST_MODE === 'true';
 }
 
+let appsSourceStorageType;
+let appsSourceStorageFilesystemPath;
+
 export class AppServerOrchestrator {
 	constructor() {
 		this._isInitialized = false;
 	}
 
 	initialize() {
+		if (this._isInitialized) {
+			return;
+		}
+
 		this._rocketchatLogger = new Logger('Rocket.Chat Apps');
 		Permissions.create('manage-apps', ['admin']);
 
@@ -38,6 +45,7 @@ export class AppServerOrchestrator {
 		this._persistModel = new AppsPersistenceModel();
 		this._storage = new AppRealStorage(this._model);
 		this._logStorage = new AppRealLogsStorage(this._logModel);
+		this._appSourceStorage = new ConfigurableAppSourceStorage(appsSourceStorageType, appsSourceStorageFilesystemPath);
 
 		this._converters = new Map();
 		this._converters.set('messages', new AppMessagesConverter(this));
@@ -50,7 +58,12 @@ export class AppServerOrchestrator {
 
 		this._bridges = new RealAppBridges(this);
 
-		this._manager = new AppManager(this._storage, this._logStorage, this._bridges);
+		this._manager = new AppManager({
+			metadataStorage: this._storage,
+			logStorage: this._logStorage,
+			bridges: this._bridges,
+			sourceStorage: this._appSourceStorage,
+		});
 
 		this._communicators = new Map();
 		this._communicators.set('methods', new AppMethods(this));
@@ -97,6 +110,10 @@ export class AppServerOrchestrator {
 		return this._manager.getExternalComponentManager().getProvidedComponents();
 	}
 
+	getAppSourceStorage() {
+		return this._appSourceStorage;
+	}
+
 	isInitialized() {
 		return this._isInitialized;
 	}
@@ -215,9 +232,48 @@ settings.addGroup('General', function() {
 			public: true,
 			hidden: false,
 		});
+
+		this.add('Apps_Framework_Source_Package_Storage_Type', 'gridfs', {
+			type: 'select',
+			values: [{
+				key: 'gridfs',
+				i18nLabel: 'GridFS',
+			}, {
+				key: 'filesystem',
+				i18nLabel: 'FileSystem',
+			}],
+			public: true,
+			hidden: false,
+			alert: 'Apps_Framework_Source_Package_Storage_Type_Alert',
+		});
+
+		this.add('Apps_Framework_Source_Package_Storage_FileSystem_Path', '', {
+			type: 'string',
+			public: true,
+			enableQuery: {
+				_id: 'Apps_Framework_Source_Package_Storage_Type',
+				value: 'filesystem',
+			},
+			alert: 'Apps_Framework_Source_Package_Storage_FileSystem_Alert',
+		});
 	});
 });
 
+settings.get('Apps_Framework_Source_Package_Storage_Type', (_, value) => {
+	if (!Apps.isInitialized()) {
+		appsSourceStorageType = value;
+	} else {
+		Apps.getAppSourceStorage().setStorage(value);
+	}
+});
+
+settings.get('Apps_Framework_Source_Package_Storage_FileSystem_Path', (_, value) => {
+	if (!Apps.isInitialized()) {
+		appsSourceStorageFilesystemPath = value;
+	} else {
+		Apps.getAppSourceStorage().setFileSystemStoragePath(value);
+	}
+});
 
 settings.get('Apps_Framework_enabled', (key, isEnabled) => {
 	// In case this gets called before `Meteor.startup`

app/apps/server/storage/AppFileSystemSourceStorage.ts

@@ -0,0 +1,68 @@
+import { promises as fs } from 'fs';
+import { join, normalize } from 'path';
+
+import { AppSourceStorage, IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
+
+export class AppFileSystemSourceStorage extends AppSourceStorage {
+	private pathPrefix = 'fs:/';
+
+	private path: string;
+
+	public setPath(path: string): void {
+		this.path = path;
+	}
+
+	public checkPath(): void {
+		if (!this.path) {
+			throw new Error('Invalid path configured for file system App storage');
+		}
+	}
+
+	public async store(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		this.checkPath();
+
+		const filePath = this.itemToFilename(item);
+
+		await fs.writeFile(filePath, zip);
+
+		return this.filenameToSourcePath(filePath);
+	}
+
+	public async fetch(item: IAppStorageItem): Promise<Buffer> {
+		if (!item.sourcePath) {
+			throw new Error('Invalid source path');
+		}
+
+		return fs.readFile(this.sourcePathToFilename(item.sourcePath));
+	}
+
+	public async update(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		this.checkPath();
+
+		const filePath = this.itemToFilename(item);
+
+		await fs.writeFile(filePath, zip);
+
+		return this.filenameToSourcePath(filePath);
+	}
+
+	public async remove(item: IAppStorageItem): Promise<void> {
+		if (!item.sourcePath) {
+			return;
+		}
+
+		return fs.unlink(this.sourcePathToFilename(item.sourcePath));
+	}
+
+	private itemToFilename(item: IAppStorageItem): string {
+		return `${ normalize(join(this.path, item.id)) }.zip`;
+	}
+
+	private filenameToSourcePath(filename: string): string {
+		return this.pathPrefix + filename;
+	}
+
+	private sourcePathToFilename(sourcePath: string): string {
+		return sourcePath.substring(this.pathPrefix.length);
+	}
+}

app/apps/server/storage/AppGridFSSourceStorage.ts

@@ -0,0 +1,81 @@
+import { MongoInternals } from 'meteor/mongo';
+import { GridFSBucket, GridFSBucketWriteStream, ObjectId } from 'mongodb';
+import { AppSourceStorage, IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
+
+import { streamToBuffer } from '../../../file-upload/server/lib/streamToBuffer';
+
+export class AppGridFSSourceStorage extends AppSourceStorage {
+	private pathPrefix = 'GridFS:/';
+
+	private bucket: GridFSBucket;
+
+	constructor() {
+		super();
+
+		const { GridFSBucket } = MongoInternals.NpmModules.mongodb.module;
+		const { db } = MongoInternals.defaultRemoteCollectionDriver().mongo;
+
+		this.bucket = new GridFSBucket(db, {
+			bucketName: 'rocketchat_apps_packages',
+			chunkSizeBytes: 1024 * 255,
+		});
+	}
+
+	public async store(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		return new Promise((resolve, reject) => {
+			const filename = this.itemToFilename(item);
+			const writeStream: GridFSBucketWriteStream = this.bucket.openUploadStream(filename)
+				.on('finish', () => resolve(this.idToPath(writeStream.id)))
+				.on('error', (error) => reject(error));
+
+			writeStream.write(zip);
+			writeStream.end();
+		});
+	}
+
+	public async fetch(item: IAppStorageItem): Promise<Buffer> {
+		return streamToBuffer(this.bucket.openDownloadStream(this.itemToObjectId(item)));
+	}
+
+	public async update(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		return new Promise((resolve, reject) => {
+			const fileId = this.itemToFilename(item);
+			const writeStream: GridFSBucketWriteStream = this.bucket.openUploadStream(fileId)
+				.on('finish', () => {
+					resolve(this.idToPath(writeStream.id));
+					// An error in the following line would not cause the update process to fail
+					// eslint-disable-next-line @typescript-eslint/no-empty-function
+					this.remove(item).catch(() => {});
+				})
+
+				.on('error', (error) => reject(error));
+
+			writeStream.write(zip);
+			writeStream.end();
+		});
+	}
+
+	public async remove(item: IAppStorageItem): Promise<void> {
+		return new Promise((resolve, reject) => {
+			this.bucket.delete(this.itemToObjectId(item), (error) => {
+				if (error) {
+					return reject(error);
+				}
+
+				resolve();
+			});
+		});
+	}
+
+	private itemToFilename(item: IAppStorageItem): string {
+		return `${ item.info.nameSlug }-${ item.info.version }.package`;
+	}
+
+	private idToPath(id: GridFSBucketWriteStream['id']): string {
+		return this.pathPrefix + id;
+	}
+
+	private itemToObjectId(item: IAppStorageItem): ObjectId {
+		return new ObjectId(item.sourcePath?.substring(this.pathPrefix.length));
+	}
+}

app/apps/server/storage/AppRealStorage.ts

@@ -1,12 +1,13 @@
-import { AppStorage } from '@rocket.chat/apps-engine/server/storage';
+import { AppMetadataStorage, IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
 
-export class AppRealStorage extends AppStorage {
-	constructor(data) {
+import { AppsModel } from '../../../models/server/models/apps-model';
+
+export class AppRealStorage extends AppMetadataStorage {
+	constructor(private db: AppsModel) {
 		super('mongodb');
-		this.db = data;
 	}
 
-	create(item) {
+	public create(item: IAppStorageItem): Promise<IAppStorageItem> {
 		return new Promise((resolve, reject) => {
 			item.createdAt = new Date();
 			item.updatedAt = new Date();
@@ -34,7 +35,7 @@ export class AppRealStorage extends AppStorage {
 		});
 	}
 
-	retrieveOne(id) {
+	public retrieveOne(id: string): Promise<IAppStorageItem> {
 		return new Promise((resolve, reject) => {
 			let doc;
 
@@ -48,9 +49,9 @@ export class AppRealStorage extends AppStorage {
 		});
 	}
 
-	retrieveAll() {
+	public retrieveAll(): Promise<Map<string, IAppStorageItem>> {
 		return new Promise((resolve, reject) => {
-			let docs;
+			let docs: Array<IAppStorageItem>;
 
 			try {
 				docs = this.db.find({}).fetch();
@@ -66,8 +67,8 @@ export class AppRealStorage extends AppStorage {
 		});
 	}
 
-	update(item) {
-		return new Promise((resolve, reject) => {
+	public update(item: IAppStorageItem): Promise<IAppStorageItem> {
+		return new Promise<string>((resolve, reject) => {
 			try {
 				this.db.update({ id: item.id }, item);
 				resolve(item.id);
@@ -77,7 +78,7 @@ export class AppRealStorage extends AppStorage {
 		}).then(this.retrieveOne.bind(this));
 	}
 
-	remove(id) {
+	public remove(id: string): Promise<{ success: boolean }> {
 		return new Promise((resolve, reject) => {
 			try {
 				this.db.remove({ id });

app/apps/server/storage/ConfigurableAppSourceStorage.ts

@@ -0,0 +1,53 @@
+import { AppSourceStorage, IAppStorageItem } from '@rocket.chat/apps-engine/server/storage';
+
+import { AppFileSystemSourceStorage } from './AppFileSystemSourceStorage';
+import { AppGridFSSourceStorage } from './AppGridFSSourceStorage';
+
+export class ConfigurableAppSourceStorage extends AppSourceStorage {
+	private filesystem: AppFileSystemSourceStorage;
+
+	private gridfs: AppGridFSSourceStorage;
+
+	private storage: AppSourceStorage;
+
+	constructor(readonly storageType: string, filesystemStoragePath: string) {
+		super();
+
+		this.filesystem = new AppFileSystemSourceStorage();
+		this.gridfs = new AppGridFSSourceStorage();
+
+		this.setStorage(storageType);
+		this.setFileSystemStoragePath(filesystemStoragePath);
+	}
+
+	public setStorage(type: string): void {
+		switch (type) {
+			case 'filesystem':
+				this.storage = this.filesystem;
+				break;
+			case 'gridfs':
+				this.storage = this.gridfs;
+				break;
+		}
+	}
+
+	public setFileSystemStoragePath(path: string): void {
+		this.filesystem.setPath(path);
+	}
+
+	public async store(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		return this.storage.store(item, zip);
+	}
+
+	public async fetch(item: IAppStorageItem): Promise<Buffer> {
+		return this.storage.fetch(item);
+	}
+
+	public async update(item: IAppStorageItem, zip: Buffer): Promise<string> {
+		return this.storage.update(item, zip);
+	}
+
+	public async remove(item: IAppStorageItem): Promise<void> {
+		return this.storage.remove(item);
+	}
+}

app/apps/server/storage/index.js

@@ -1,4 +1,5 @@
-import { AppRealLogsStorage } from './logs-storage';
-import { AppRealStorage } from './storage';
-
-export { AppRealLogsStorage, AppRealStorage };
+export { AppRealLogsStorage } from './logs-storage';
+export { AppRealStorage } from './AppRealStorage';
+export { AppFileSystemSourceStorage } from './AppFileSystemSourceStorage';
+export { AppGridFSSourceStorage } from './AppGridFSSourceStorage';
+export { ConfigurableAppSourceStorage } from './ConfigurableAppSourceStorage';

app/file-upload/server/lib/streamToBuffer.ts

@@ -1,11 +1,12 @@
 import { Readable } from 'stream';
 
-export const streamToBuffer = (stream: Readable): Promise<Buffer> => new Promise((resolve) => {
+export const streamToBuffer = (stream: Readable): Promise<Buffer> => new Promise((resolve, reject) => {
 	const chunks: Array<Buffer> = [];
 
 	stream
 		.on('data', (data) => chunks.push(data))
 		.on('end', () => resolve(Buffer.concat(chunks)))
+		.on('error', (error) => reject(error))
 		// force stream to resume data flow in case it was explicitly paused before
 		.resume();
 });

package-lock.json

@@ -5282,9 +5282,9 @@
       }
     },
     "@rocket.chat/apps-engine": {
-      "version": "1.28.0-alpha.5379",
-      "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.28.0-alpha.5379.tgz",
-      "integrity": "sha512-EaLK46AzhoKgJDo+frl83SmBGgcauLvz/2YKy8xBLOzhLQQYUY9IBSXDlC2B+sTMkEaefE+9wV4xA5F+q74FGg==",
+      "version": "1.28.0-alpha.5428",
+      "resolved": "https://registry.npmjs.org/@rocket.chat/apps-engine/-/apps-engine-1.28.0-alpha.5428.tgz",
+      "integrity": "sha512-M2i74yj3fvOw60FssXrF+dSQ5F9U/LOa865UhJH8ijcY1lTbqku1v7gXEi4GaTZ9HSS7bV47YECCj5qQ1ENgcw==",
       "requires": {
         "adm-zip": "^0.4.9",
         "cryptiles": "^4.1.3",

package.json

@@ -159,7 +159,7 @@
 		"@nivo/heatmap": "0.73.0",
 		"@nivo/line": "0.62.0",
 		"@nivo/pie": "0.73.0",
-		"@rocket.chat/apps-engine": "1.28.0-alpha.5379",
+		"@rocket.chat/apps-engine": "^1.28.0-alpha.5428",
 		"@rocket.chat/css-in-js": "^0.6.3-dev.322",
 		"@rocket.chat/emitter": "^0.6.3-dev.322",
 		"@rocket.chat/fuselage": "^0.6.3-dev.322",

packages/rocketchat-i18n/i18n/en.i18n.json

@@ -433,6 +433,12 @@
   "Apps_Framework_Development_Mode": "Enable development mode",
   "Apps_Framework_Development_Mode_Description": "Development mode allows the installation of Apps that are not from the Rocket.Chat's Marketplace.",
   "Apps_Framework_enabled": "Enable the App Framework",
+  "Apps_Framework_Source_Package_Storage_Type":"Apps' Source Package Storage type",
+  "Apps_Framework_Source_Package_Storage_Type_Description":"Choose where all the apps' source code will be stored. Apps can have multiple megabytes in size each.",
+  "Apps_Framework_Source_Package_Storage_Type_Alert":"Changing where the apps are stored may cause instabilities in apps there are already installed",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Path":"Directory for storing apps source package",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Path_Description":"Absolute path in the filesystem for storing the apps' source code (in zip file format)",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Alert":"Make sure the chosen directory exist and Rocket.Chat can access it (e.g. permission to read/write)",
   "Apps_Game_Center": "Game Center",
   "Apps_Game_Center_Back": "Back to Game Center",
   "Apps_Game_Center_Invite_Friends": "Invite your friends to join",

packages/rocketchat-i18n/i18n/pt-BR.i18n.json

@@ -433,6 +433,12 @@
   "Apps_Framework_Development_Mode": "Habilitar modo de desenvolvimento",
   "Apps_Framework_Development_Mode_Description": "O modo de desenvolvimento permite a instalação de aplicativos que não são do Marketplace do Rocket.Chat.",
   "Apps_Framework_enabled": "Ativar o App Framework",
+  "Apps_Framework_Source_Package_Storage_Type":"Tipo de armazenamento pata o código fonte dos apps",
+  "Apps_Framework_Source_Package_Storage_Type_Description":"Escolha como o código fonte dos apps serão armazenamos. Apps podem ter vários megabytes em tamanho.",
+  "Apps_Framework_Source_Package_Storage_Type_Alert":"Mudar o tipo de armazenamento dos apps pode causar instabilidade em apps que já estão instalados.",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Path":"Diretório de armazenamento de código fonte dos apps",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Path_Description":"Caminho absoluto no sistema de arquivo para armazenar o código fonte dos apps (em formato zip).",
+  "Apps_Framework_Source_Package_Storage_FileSystem_Alert":"Certifique que o diretório escolhido existe a o Rocket.Chat pode accessá-lo (por exemplo, permissão de escrita/leitura).",
   "Apps_Game_Center": "Game Center",
   "Apps_Game_Center_Back": "Voltar ao Game Center",
   "Apps_Game_Center_Invite_Friends": "Convide seus amigos para entrar",

server/main.d.ts

@@ -1,5 +1,6 @@
 import { EJSON } from 'meteor/ejson';
 import { Db, Collection } from 'mongodb';
+import * as mongodb from 'mongodb';
 
 import { IStreamer, IStreamerConstructor } from './modules/streamer/streamer.module';
 
@@ -109,6 +110,13 @@ declare module 'meteor/mongo' {
 	}
 
 	namespace MongoInternals {
+		export const NpmModules: {
+			mongodb: {
+				version: string;
+				module: typeof mongodb;
+			};
+		};
+
 		function defaultRemoteCollectionDriver(): RemoteCollectionDriver;
 
 		class ConnectionClass {}

server/startup/migrations/index.ts

@@ -61,4 +61,5 @@ import './v234';
 import './v235';
 import './v236';
 import './v237';
+import './v238';
 import './xrun';

server/startup/migrations/v238.ts

@@ -0,0 +1,19 @@
+import { AppManager } from '@rocket.chat/apps-engine/server/AppManager';
+
+import { addMigration } from '../../lib/migrations';
+import { Apps } from '../../../app/apps/server';
+
+addMigration({
+	version: 238,
+	up() {
+		Apps.initialize();
+
+		const apps = Apps._model.find().fetch();
+
+		for (const app of apps) {
+			const zipFile = Buffer.from(app.zip, 'base64');
+			Promise.await((Apps._manager as AppManager).update(zipFile, app.permissionsGranted, { loadApp: false }));
+			Promise.await(Apps._model.update({ id: app.id }, { $unset: { zip: 1, compiled: 1 } }));
+		}
+	},
+});