[IMPROVE] Hipchat Enterprise Importer (#12985)

* Added message showing the file size limit on the importers

* Importer improvements

* Added missing reference

* Removed globals

* Fixed data importers

* Fixed import

* Removed log

* Changed hipchat enterprise importer to prepare files from the file system

* Use a file URL on the importer

* Avoid duplicated emails

* Prevent duplicated emails from crashing the import process

* Identify errors on the import process

* Fixed attachment import

* Fixed problem with invalid users when creating channels

* Added labels for checkboxes

packages/rocketchat-api/server/index.js

@@ -20,6 +20,7 @@ import './v1/emoji-custom';
 import './v1/groups';
 import './v1/im';
 import './v1/integrations';
+import './v1/import';
 import './v1/misc';
 import './v1/permissions';
 import './v1/push';

packages/rocketchat-api/server/v1/import.js

@@ -0,0 +1,42 @@
+import { Meteor } from 'meteor/meteor';
+import { RocketChat } from 'meteor/rocketchat:lib';
+
+RocketChat.API.v1.addRoute('uploadImportFile', { authRequired: true }, {
+	post() {
+		const { binaryContent, contentType, fileName, importerKey } = this.bodyParams;
+
+		Meteor.runAsUser(this.userId, () => {
+			RocketChat.API.v1.success(Meteor.call('uploadImportFile', binaryContent, contentType, fileName, importerKey));
+		});
+
+		return RocketChat.API.v1.success();
+	},
+
+});
+
+RocketChat.API.v1.addRoute('downloadPublicImportFile', { authRequired: true }, {
+	post() {
+		const { fileUrl, importerKey } = this.bodyParams;
+
+		Meteor.runAsUser(this.userId, () => {
+			RocketChat.API.v1.success(Meteor.call('downloadPublicImportFile', fileUrl, importerKey));
+		});
+
+		return RocketChat.API.v1.success();
+	},
+
+});
+
+RocketChat.API.v1.addRoute('getImportFileData', { authRequired: true }, {
+	get() {
+		const { importerKey } = this.requestParams();
+		let result;
+		Meteor.runAsUser(this.userId, () => {
+			result = Meteor.call('getImportFileData', importerKey);
+		});
+
+		return RocketChat.API.v1.success(result);
+	},
+
+});
+

packages/rocketchat-i18n/i18n/en.i18n.json

@@ -1000,6 +1000,7 @@
   "Duplicate_archived_channel_name": "An archived Channel with name `#%s` exists",
   "Duplicate_archived_private_group_name": "An archived Private Group with name '%s' exists",
   "Duplicate_channel_name": "A Channel with name '%s' exists",
+  "Duplicated_Email_address_will_be_ignored": "Duplicated email address will be ignored.",
   "Duplicate_private_group_name": "A Private Group with name '%s' exists",
   "Duration": "Duration",
   "E2E Encryption": "E2E Encryption",
@@ -1270,6 +1271,9 @@
   "FileUpload_Webdav_Proxy_Uploads_Description": "Proxy upload file transmissions through your server instead of direct access to the asset's URL",
   "files": "files",
   "Files_only": "Only remove the attached files, keep messages",
+  "FileSize_KB": "__fileSize__ KB",
+  "FileSize_MB": "__fileSize__ MB",
+  "FileSize_Bytes": "__fileSize__ Bytes",
   "Financial_Services": "Financial Services",
   "First_Channel_After_Login": "First Channel After Login",
   "First_response_time": "First Response Time",
@@ -1425,7 +1429,9 @@
   "Importer_setup_error": "An error occurred while setting up the importer.",
   "Importer_Slack_Users_CSV_Information": "The file uploaded must be Slack's Users export file, which is a CSV file. See here for more information:",
   "Importer_Source_File": "Source File Selection",
-  "Inclusive": "Inclusive",
+  "Importer_Upload_FileSize_Message": "Your server settings allow the upload of files of any size up to __maxFileSize__.",
+  "Importer_Upload_Unlimited_FileSize": "Your server settings allow the upload of files of any size.",
+  "Importer_ExternalUrl_Description": "You can also use an URL for a publicly accessible file:",  "Inclusive": "Inclusive",
   "Incoming_Livechats": "Incoming Livechats",
   "Incoming_WebHook": "Incoming WebHook",
   "Industry": "Industry",

packages/rocketchat-importer-hipchat-enterprise/server/importer.js

@@ -12,6 +12,7 @@ import { RocketChat } from 'meteor/rocketchat:lib';
 import { Readable } from 'stream';
 import path from 'path';
 import s from 'underscore.string';
+import fs from 'fs';
 import TurndownService from 'turndown';
 
 const turndownService = new TurndownService({
@@ -43,11 +44,10 @@ export class HipChatEnterpriseImporter extends Base {
 		this.directMessages = new Map();
 	}
 
-	prepare(dataURI, sentContentType, fileName) {
-		super.prepare(dataURI, sentContentType, fileName);
-
+	prepareUsingLocalFile(fullFilePath) {
 		const tempUsers = [];
 		const tempRooms = [];
+		const emails = [];
 		const tempMessages = new Map();
 		const tempDirectMessages = new Map();
 		const promise = new Promise((resolve, reject) => {
@@ -72,10 +72,7 @@ export class HipChatEnterpriseImporter extends Base {
 					if (info.base === 'users.json') {
 						super.updateProgress(ProgressStep.PREPARING_USERS);
 						for (const u of file) {
-							// if (!u.User.email) {
-							// 	// continue;
-							// }
-							tempUsers.push({
+							const userData = {
 								id: u.User.id,
 								email: u.User.email,
 								name: u.User.name,
@@ -83,7 +80,17 @@ export class HipChatEnterpriseImporter extends Base {
 								avatar: u.User.avatar && u.User.avatar.replace(/\n/g, ''),
 								timezone: u.User.timezone,
 								isDeleted: u.User.is_deleted,
-							});
+							};
+
+							if (u.User.email) {
+								if (emails.indexOf(u.User.email) >= 0) {
+									userData.is_email_taken = true;
+								} else {
+									emails.push(u.User.email);
+								}
+							}
+
+							tempUsers.push(userData);
 						}
 					} else if (info.base === 'rooms.json') {
 						super.updateProgress(ProgressStep.PREPARING_CHANNELS);
@@ -129,6 +136,8 @@ export class HipChatEnterpriseImporter extends Base {
 										userId: m.UserMessage.sender.id,
 										text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`,
 										ts: new Date(m.UserMessage.timestamp.split(' ')[0]),
+										attachment: m.UserMessage.attachment,
+										attachment_path: m.UserMessage.attachment_path,
 									});
 								} else if (m.NotificationMessage) {
 									const text = m.NotificationMessage.message.indexOf('/me ') === -1 ? m.NotificationMessage.message : `${ m.NotificationMessage.message.replace(/\/me /, '_') }_`;
@@ -140,6 +149,8 @@ export class HipChatEnterpriseImporter extends Base {
 										alias: m.NotificationMessage.sender,
 										text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text,
 										ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]),
+										attachment: m.NotificationMessage.attachment,
+										attachment_path: m.NotificationMessage.attachment_path,
 									});
 								} else if (m.TopicRoomMessage) {
 									roomMsgs.push({
@@ -174,6 +185,23 @@ export class HipChatEnterpriseImporter extends Base {
 			});
 
 			this.extract.on('finish', Meteor.bindEnvironment(() => {
+				// Check if any of the emails used are already taken
+				if (emails.length > 0) {
+					const conflictingUsers = RocketChat.models.Users.find({ 'emails.address': { $in: emails } });
+
+					conflictingUsers.forEach((conflictingUser) => tempUsers.forEach((newUser) => conflictingUser.emails.forEach((email) => {
+						if (email && email.address === newUser.email) {
+							if (conflictingUser.username !== newUser.username) {
+								newUser.is_email_taken = true;
+								newUser.do_import = false;
+								return false;
+							}
+						}
+
+						return true;
+					})));
+				}
+
 				// Insert the users record, eventually this might have to be split into several ones as well
 				// if someone tries to import a several thousands users instance
 				const usersId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', users: tempUsers });
@@ -240,7 +268,7 @@ export class HipChatEnterpriseImporter extends Base {
 					return;
 				}
 
-				const selectionUsers = tempUsers.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true));
+				const selectionUsers = tempUsers.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted, false, u.do_import !== false, u.is_email_taken === true));
 				const selectionChannels = tempRooms.map((r) => new SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate));
 				const selectionMessages = this.importRecord.count.messages;
 
@@ -249,64 +277,17 @@ export class HipChatEnterpriseImporter extends Base {
 				resolve(new Selection(this.name, selectionUsers, selectionChannels, selectionMessages));
 			}));
 
-			// Wish I could make this cleaner :(
-			const split = dataURI.split(',');
-			const read = new this.Readable;
-			read.push(new Buffer(split[split.length - 1], 'base64'));
-			read.push(null);
-			read.pipe(this.zlib.createGunzip()).pipe(this.extract);
+			const rs = fs.createReadStream(fullFilePath);
+
+			rs.pipe(this.zlib.createGunzip()).pipe(this.extract);
 		});
 
 		return promise;
 	}
 
-	startImport(importSelection) {
-		super.startImport(importSelection);
-		const started = Date.now();
-
-		// Ensure we're only going to import the users that the user has selected
-		for (const user of importSelection.users) {
-			for (const u of this.users.users) {
-				if (u.id === user.user_id) {
-					u.do_import = user.do_import;
-				}
-			}
-		}
-		this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } });
-
-		// Ensure we're only importing the channels the user has selected.
-		for (const channel of importSelection.channels) {
-			for (const c of this.channels.channels) {
-				if (c.id === channel.channel_id) {
-					c.do_import = channel.do_import;
-				}
-			}
-		}
-		this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } });
-
-		const startedByUserId = Meteor.userId();
-		Meteor.defer(() => {
-			super.updateProgress(ProgressStep.IMPORTING_USERS);
-
-			try {
-				// Import the users
-				for (const u of this.users.users) {
-					this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`);
-					if (!u.do_import) {
-						continue;
-					}
-
+	_importUser(u, startedByUserId) {
 		Meteor.runAsUser(startedByUserId, () => {
-						let existantUser;
-
-						if (u.email) {
-							RocketChat.models.Users.findOneByEmailAddress(u.email);
-						}
-
-						// If we couldn't find one by their email address, try to find an existing user by their username
-						if (!existantUser) {
-							existantUser = RocketChat.models.Users.findOneByUsername(u.username);
-						}
+			const existantUser = RocketChat.models.Users.findOneByUsername(u.username);
 
 			if (existantUser) {
 				// since we have an existing user, let's try a few things
@@ -314,11 +295,15 @@ export class HipChatEnterpriseImporter extends Base {
 				RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } });
 			} else {
 				const user = { email: u.email, password: Random.id() };
+				// if (u.is_email_taken && u.email) {
+				// 	user.email = user.email.replace('@', `+rocket.chat_${ Math.floor(Math.random() * 10000).toString() }@`);
+				// }
 				if (!user.email) {
 					delete user.email;
 					user.username = u.username;
 				}
 
+				try {
 					const userId = Accounts.createUser(user);
 					Meteor.runAsUser(userId, () => {
 						Meteor.call('setUsername', u.username, { joinDefaultChannelsSilenced: true });
@@ -338,15 +323,80 @@ export class HipChatEnterpriseImporter extends Base {
 						RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } });
 						u.rocketId = userId;
 					});
+				} catch (e) {
+					this.addUserError(u.id, e);
+				}
 			}
 
 			super.addCountCompleted(1);
 		});
 	}
+
+	startImport(importSelection) {
+		super.startImport(importSelection);
+		const started = Date.now();
+
+		// Ensure we're only going to import the users that the user has selected
+		for (const user of importSelection.users) {
+			for (const u of this.users.users) {
+				if (u.id === user.user_id) {
+					u.do_import = user.do_import;
+				}
+			}
+		}
 		this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } });
 
-				// Import the channels
+		// Ensure we're only importing the channels the user has selected.
+		for (const channel of importSelection.channels) {
+			for (const c of this.channels.channels) {
+				if (c.id === channel.channel_id) {
+					c.do_import = channel.do_import;
+				}
+			}
+		}
+		this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } });
+
+		const startedByUserId = Meteor.userId();
+		Meteor.defer(() => {
+			try {
+				super.updateProgress(ProgressStep.IMPORTING_USERS);
+				this._importUsers(startedByUserId);
+
 				super.updateProgress(ProgressStep.IMPORTING_CHANNELS);
+				this._importChannels(startedByUserId);
+
+				super.updateProgress(ProgressStep.IMPORTING_MESSAGES);
+				this._importMessages(startedByUserId);
+				this._importDirectMessages();
+
+				// super.updateProgress(ProgressStep.FINISHING);
+				super.updateProgress(ProgressStep.DONE);
+			} catch (e) {
+				super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) });
+				this.logger.error(e);
+				super.updateProgress(ProgressStep.ERROR);
+			}
+
+			const timeTook = Date.now() - started;
+			this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`);
+		});
+
+		return super.getProgress();
+	}
+
+	_importUsers(startedByUserId) {
+		for (const u of this.users.users) {
+			this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`);
+			if (!u.do_import) {
+				continue;
+			}
+
+			this._importUser(u, startedByUserId);
+		}
+		this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } });
+	}
+
+	_importChannels(startedByUserId) {
 		for (const c of this.channels.channels) {
 			if (!c.do_import) {
 				continue;
@@ -362,46 +412,58 @@ export class HipChatEnterpriseImporter extends Base {
 					// Find the rocketchatId of the user who created this channel
 					let creatorId = startedByUserId;
 					for (const u of this.users.users) {
-								if (u.id === c.creator && u.do_import) {
+						if (u.id === c.creator && u.do_import && u.rocketId) {
 							creatorId = u.rocketId;
+							break;
 						}
 					}
 
 					// Create the channel
 					Meteor.runAsUser(creatorId, () => {
+						try {
 							const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, []);
 							c.rocketId = roomInfo.rid;
+						} catch (e) {
+							this.logger.error(`Failed to create channel, using userId: ${ creatorId };`, e);
+						}
 					});
 
+					if (c.rocketId) {
 						RocketChat.models.Rooms.update({ _id: c.rocketId }, { $set: { ts: c.created, topic: c.topic }, $addToSet: { importIds: c.id } });
 					}
+				}
 
 				super.addCountCompleted(1);
 			});
 		}
 		this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } });
+	}
 
-				// Import the Messages
-				super.updateProgress(ProgressStep.IMPORTING_MESSAGES);
-				for (const [ch, messagesMap] of this.messages.entries()) {
-					const hipChannel = this.getChannelFromRoomIdentifier(ch);
-					if (!hipChannel.do_import) {
-						continue;
+	_importAttachment(msg, room, sender) {
+		if (msg.attachment_path) {
+			const details = {
+				message_id: `${ msg.id }-attachment`,
+				name: msg.attachment.name,
+				size: msg.attachment.size,
+				userId: sender._id,
+				rid: room._id,
+			};
+
+			this.uploadFile(details, msg.attachment.url, sender, room, msg.ts);
+		}
 	}
 
-					const room = RocketChat.models.Rooms.findOneById(hipChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } });
-					Meteor.runAsUser(startedByUserId, () => {
-						for (const [msgGroupData, msgs] of messagesMap.entries()) {
-							super.updateRecord({ messagesstatus: `${ ch }/${ msgGroupData }.${ msgs.messages.length }` });
-							for (const msg of msgs.messages) {
+	_importSingleMessage(msg, ch, msgGroupData, room) {
 		if (isNaN(msg.ts)) {
 			this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`);
 			super.addCountCompleted(1);
-									continue;
+			return;
 		}
 
 		const creator = this.getRocketUserFromUserId(msg.userId);
 		if (creator) {
+			this._importAttachment(msg, room, creator);
+
 			switch (msg.type) {
 				case 'user':
 					RocketChat.sendMessage(creator, {
@@ -424,11 +486,27 @@ export class HipChatEnterpriseImporter extends Base {
 
 		super.addCountCompleted(1);
 	}
+
+	_importMessages(startedByUserId) {
+		for (const [ch, messagesMap] of this.messages.entries()) {
+			const hipChannel = this.getChannelFromRoomIdentifier(ch);
+			if (!hipChannel.do_import) {
+				continue;
+			}
+
+			const room = RocketChat.models.Rooms.findOneById(hipChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } });
+			Meteor.runAsUser(startedByUserId, () => {
+				for (const [msgGroupData, msgs] of messagesMap.entries()) {
+					super.updateRecord({ messagesstatus: `${ ch }/${ msgGroupData }.${ msgs.messages.length }` });
+					for (const msg of msgs.messages) {
+						this._importSingleMessage(msg, ch, msgGroupData, room);
+					}
 				}
 			});
 		}
+	}
 
-				// Import the Direct Messages
+	_importDirectMessages() {
 		for (const [directMsgRoom, directMessagesMap] of this.directMessages.entries()) {
 			const hipUser = this.getUserFromDirectMessageIdentifier(directMsgRoom);
 			if (!hipUser || !hipUser.do_import) {
@@ -472,7 +550,7 @@ export class HipChatEnterpriseImporter extends Base {
 					Meteor.runAsUser(sender._id, () => {
 						if (msg.attachment_path) {
 							const details = {
-										message_id: msg.id,
+								message_id: `${ msg.id }-attachment`,
 								name: msg.attachment.name,
 								size: msg.attachment.size,
 								userId: sender._id,
@@ -495,24 +573,10 @@ export class HipChatEnterpriseImporter extends Base {
 				}
 			}
 		}
-
-				super.updateProgress(ProgressStep.FINISHING);
-				super.updateProgress(ProgressStep.DONE);
-			} catch (e) {
-				super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) });
-				this.logger.error(e);
-				super.updateProgress(ProgressStep.ERROR);
-			}
-
-			const timeTook = Date.now() - started;
-			this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`);
-		});
-
-		return super.getProgress();
 	}
 
 	getSelection() {
-		const selectionUsers = this.users.users.map((u) => new SelectionUser(u.id, u.username, u.email, false, false, true));
+		const selectionUsers = this.users.users.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted === true, false, u.do_import !== false, u.is_email_taken === true));
 		const selectionChannels = this.channels.channels.map((c) => new SelectionChannel(c.id, c.name, false, true, c.isPrivate));
 		const selectionMessages = this.importRecord.count.messages;
 

packages/rocketchat-importer-hipchat/server/importer.js

@@ -24,8 +24,8 @@ export class HipChatImporter extends Base {
 		this.usersPrefix = 'hipchat_export/users/';
 	}
 
-	prepare(dataURI, sentContentType, fileName) {
-		super.prepare(dataURI, sentContentType, fileName);
+	prepare(dataURI, sentContentType, fileName, skipTypeCheck) {
+		super.prepare(dataURI, sentContentType, fileName, skipTypeCheck);
 		const { image } = RocketChatFile.dataURIParse(dataURI);
 		// const contentType = ref.contentType;
 		const zip = new this.AdmZip(new Buffer(image, 'base64'));

packages/rocketchat-importer/client/admin/adminImport.html

@@ -12,6 +12,7 @@
 								<h1>{{ name }}</h1>
 								<div class="section-content">
 									<div>{{getDescription .}}</div>
+									<br/>
 									<button class="button primary start-import">{{_ "Start"}}</button>
 								</div>
 							</div>

packages/rocketchat-importer/client/admin/adminImportPrepare.html

@@ -36,8 +36,10 @@
 												{{#each users}}
 													{{#unless is_bot}}
 														<li>
-															<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
+															<input type="checkbox" name="{{user_id}}" id="user_{{user_id}}" checked="checked" />
+															<label for="user_{{user_id}}">{{username}} - {{email}}</label>
 															{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
+															{{#if is_email_taken }} <em>({{_ "Duplicated_Email_address_will_be_ignored"}})</em>{{/if}}
 														</li>
 													{{/unless}}
 												{{/each}}
@@ -53,7 +55,8 @@
 											<ul>
 												{{#each channels}}
 													<li>
-														<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
+														<input type="checkbox" name="{{channel_id}}" id="channel_{{channel_id}}" checked="checked" />
+														<label for="channel_{{channel_id}}">{{name}}</label>
 														{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
 														{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
 													</li>
@@ -72,9 +75,19 @@
 								{{else}}
 									<div class="section">
 										<h1>{{_ "Importer_Source_File"}}</h1>
+										<div class="section-content">
+											{{ fileSizeLimitMessage }}
+										</div>
 										<div class="section-content">
 											<input type="file" class="import-file-input">
 										</div>
+										<div class="section-content">
+											{{_ "Importer_ExternalUrl_Description"}}
+										</div>
+										<div class="section-content">
+											<input type="text" class="import-file-url" size=50 placeholder="{{_ "File URL" }}">
+											<input type="button" value="{{_ "Import" }}" class="download-public-url">
+										</div>
 									</div>
 								{{/if}}
 							{{/if}}

packages/rocketchat-importer/client/admin/adminImportPrepare.js

@@ -3,6 +3,7 @@ import { ReactiveVar } from 'meteor/reactive-var';
 import { Importers } from 'meteor/rocketchat:importer';
 import { FlowRouter } from 'meteor/kadira:flow-router';
 import { Template } from 'meteor/templating';
+import { TAPi18n } from 'meteor/tap:i18n';
 import { RocketChat, handleError } from 'meteor/rocketchat:lib';
 import { t } from 'meteor/rocketchat:utils';
 import toastr from 'toastr';
@@ -31,32 +32,34 @@ Template.adminImportPrepare.helpers({
 	message_count() {
 		return Template.instance().message_count.get();
 	},
-});
+	fileSizeLimitMessage() {
+		const maxFileSize = RocketChat.settings.get('FileUpload_MaxFileSize');
+		let message;
 
-Template.adminImportPrepare.events({
-	'change .import-file-input'(event, template) {
-		const importer = this;
-		if (!importer || !importer.key) { return; }
+		if (maxFileSize > 0) {
+			const sizeInKb = maxFileSize / 1024;
+			const sizeInMb = sizeInKb / 1024;
 
-		const e = event.originalEvent || event;
-		let { files } = e.target;
-		if (!files || (files.length === 0)) {
-			files = (e.dataTransfer != null ? e.dataTransfer.files : undefined) || [];
+			let fileSizeMessage;
+			if (sizeInMb > 0) {
+				fileSizeMessage = TAPi18n.__('FileSize_MB', { fileSize: sizeInMb.toFixed(2) });
+			} else if (sizeInKb > 0) {
+				fileSizeMessage = TAPi18n.__('FileSize_KB', { fileSize: sizeInKb.toFixed(2) });
+			} else {
+				fileSizeMessage = TAPi18n.__('FileSize_Bytes', { fileSize: maxFileSize.toFixed(0) });
 			}
 
-		Array.from(files).forEach((file) => {
-			template.preparing.set(true);
-
-			const reader = new FileReader();
-			reader.readAsDataURL(file);
-			reader.onloadend = () => {
-				Meteor.call('prepareImport', importer.key, reader.result, file.type, file.name, function(error, data) {
-					if (error) {
-						toastr.error(t('Invalid_Import_File_Type'));
-						template.preparing.set(false);
-						return;
+			message = TAPi18n.__('Importer_Upload_FileSize_Message', { maxFileSize: fileSizeMessage });
+		} else {
+			message = TAPi18n.__('Importer_Upload_Unlimited_FileSize');
 		}
 
+		return message;
+	},
+});
+
+function getImportFileData(importer, template) {
+	RocketChat.API.get(`v1/getImportFileData?importerKey=${ importer.key }`).then((data) => {
 		if (!data) {
 			console.warn(`The importer ${ importer.key } is not set up correctly, as it did not return any data.`);
 			toastr.error(t('Importer_not_setup'));
@@ -64,6 +67,13 @@ Template.adminImportPrepare.events({
 			return;
 		}
 
+		if (data.waiting) {
+			setTimeout(() => {
+				getImportFileData(importer, template);
+			}, 500);
+			return;
+		}
+
 		if (data.step) {
 			console.warn('Invalid file, contains `data.step`.', data);
 			toastr.error(t('Invalid_Export_File', importer.key));
@@ -76,11 +86,73 @@ Template.adminImportPrepare.events({
 		template.message_count.set(data.message_count);
 		template.loaded.set(true);
 		template.preparing.set(false);
+	}).catch((error) => {
+		if (error) {
+			toastr.error(t('Failed_To_Load_Import_Data'));
+			template.preparing.set(false);
+		}
+	});
+}
+
+
+Template.adminImportPrepare.events({
+	'change .import-file-input'(event, template) {
+		const importer = this;
+		if (!importer || !importer.key) { return; }
+
+		const e = event.originalEvent || event;
+		let { files } = e.target;
+		if (!files || (files.length === 0)) {
+			files = (e.dataTransfer != null ? e.dataTransfer.files : undefined) || [];
+		}
+
+		Array.from(files).forEach((file) => {
+			template.preparing.set(true);
+
+			const reader = new FileReader();
+
+			reader.readAsBinaryString(file);
+			reader.onloadend = () => {
+				RocketChat.API.post('v1/uploadImportFile', {
+					binaryContent: reader.result,
+					contentType: file.type,
+					fileName: file.name,
+					importerKey: importer.key,
+				}).then(() => {
+					getImportFileData(importer, template);
+				}).catch((error) => {
+					if (error) {
+						toastr.error(t('Failed_To_upload_Import_File'));
+						template.preparing.set(false);
+					}
 				});
 			};
 		});
 	},
 
+
+	'click .download-public-url'(event, template) {
+		const importer = this;
+		if (!importer || !importer.key) { return; }
+
+		const fileUrl = $('.import-file-url').val();
+
+		template.preparing.set(true);
+
+		RocketChat.API.post('v1/downloadPublicImportFile', {
+			fileUrl,
+			importerKey: importer.key,
+		}).then(() => {
+			getImportFileData(importer, template);
+		}).catch((error) => {
+			if (error) {
+				toastr.error(t('Failed_To_upload_Import_File'));
+				template.preparing.set(false);
+			}
+		});
+	},
+
+
 	'click .button.start'(event, template) {
 		const btn = this;
 		$(btn).prop('disabled', true);
@@ -127,7 +199,6 @@ Template.adminImportPrepare.events({
 	},
 });
 
-
 Template.adminImportPrepare.onCreated(function() {
 	const instance = this;
 	this.preparing = new ReactiveVar(true);

packages/rocketchat-importer/lib/ImporterProgressStep.js

@@ -1,6 +1,9 @@
 /** The progress step that an importer is at. */
 export const ProgressStep = Object.freeze({
+	UPLOADING: 'importer_uploading',
 	NEW: 'importer_new',
+	DOWNLOADING_FILE_URL: 'downloading_file_url',
+	DOWNLOAD_COMPLETE: 'download_complete',
 	PREPARING_STARTED: 'importer_preparing_started',
 	PREPARING_USERS: 'importer_preparing_users',
 	PREPARING_CHANNELS: 'importer_preparing_channels',

packages/rocketchat-importer/package.js

@@ -14,6 +14,7 @@ Package.onUse(function(api) {
 		'rocketchat:lib',
 		'rocketchat:logger',
 	]);
+
 	api.mainModule('client/index.js', 'client');
 	api.mainModule('server/index.js', 'server');
 });

packages/rocketchat-importer/server/classes/ImporterBase.js

@@ -10,6 +10,7 @@ import { RocketChat } from 'meteor/rocketchat:lib';
 import { Logger } from 'meteor/rocketchat:logger';
 import { FileUpload } from 'meteor/rocketchat:file-upload';
 import http from 'http';
+import fs from 'fs';
 import https from 'https';
 import AdmZip from 'adm-zip';
 import getFileType from 'file-type';
@@ -98,8 +99,16 @@ export class Base {
 		this.progress = new Progress(this.info.key, this.info.name);
 		this.collection = RawImports;
 
-		const importId = Imports.insert({ type: this.info.name, ts: Date.now(), status: this.progress.step, valid: true, user: Meteor.user()._id });
+		const userId = Meteor.user()._id;
+		const importRecord = Imports.findPendingImport(this.info.key);
+
+		if (importRecord) {
+			this.importRecord = importRecord;
+			this.progress.step = this.importRecord.status;
+		} else {
+			const importId = Imports.insert({ type: this.info.name, importerKey: this.info.key, ts: Date.now(), status: this.progress.step, valid: true, user: userId });
 			this.importRecord = Imports.findOne(importId);
+		}
 
 		this.users = {};
 		this.channels = {};
@@ -109,6 +118,37 @@ export class Base {
 		this.logger.debug(`Constructed a new ${ info.name } Importer.`);
 	}
 
+	/**
+	 * Registers the file name and content type on the import operation
+	 *
+	 * @param {string} fileName The name of the uploaded file.
+	 * @param {string} contentType The sent file type.
+	 * @returns {Progress} The progress record of the import.
+	 */
+	startFileUpload(fileName, contentType) {
+		this.updateProgress(ProgressStep.UPLOADING);
+		return this.updateRecord({ file: fileName, contentType });
+	}
+
+	/**
+	 * Takes the uploaded file and extracts the users, channels, and messages from it.
+	 *
+	 * @param {string} fullFilePath the full path of the uploaded file
+	 * @returns {Progress} The progress record of the import.
+	 */
+	prepareUsingLocalFile(fullFilePath) {
+		const file = fs.readFileSync(fullFilePath);
+		const buffer = Buffer.isBuffer(file) ? file : new Buffer(file);
+
+		const { contentType } = this.importRecord;
+		const fileName = this.importRecord.file;
+
+		const data = buffer.toString('base64');
+		const dataURI = `data:${ contentType };base64,${ data }`;
+
+		return this.prepare(dataURI, contentType, fileName, true);
+	}
+
 	/**
 	 * Takes the uploaded file and extracts the users, channels, and messages from it.
 	 *
@@ -249,6 +289,24 @@ export class Base {
 		return this.progress;
 	}
 
+	/**
+	 * Registers error information on a specific user from the import record
+	 *
+	 * @param {int} the user id
+	 * @param {object} an exception object
+	 */
+	addUserError(userId, error) {
+		Imports.model.update({
+			_id: this.importRecord._id,
+			'fileData.users.user_id': userId,
+		}, {
+			$set: {
+				'fileData.users.$.error': error,
+				hasErrors: true,
+			},
+		});
+	}
+
 	/**
 	 * Updates the import record with the given fields being `set`.
 	 *

packages/rocketchat-importer/server/classes/ImporterSelectionUser.js

@@ -8,13 +8,15 @@ export class SelectionUser {
 	 * @param {boolean} is_deleted whether the user was deleted or not
 	 * @param {boolean} is_bot whether the user is a bot or not
 	 * @param {boolean} do_import whether we are going to import this user or not
+	 * @param {boolean} is_email_taken whether there's an existing user with the same email
 	 */
-	constructor(user_id, username, email, is_deleted, is_bot, do_import) {
+	constructor(user_id, username, email, is_deleted, is_bot, do_import, is_email_taken = false) {
 		this.user_id = user_id;
 		this.username = username;
 		this.email = email;
 		this.is_deleted = is_deleted;
 		this.is_bot = is_bot;
 		this.do_import = do_import;
+		this.is_email_taken = is_email_taken;
 	}
 }

packages/rocketchat-importer/server/index.js

@@ -15,7 +15,11 @@ import './methods/prepareImport';
 import './methods/restartImport';
 import './methods/setupImporter';
 import './methods/startImport';
+import './methods/uploadImportFile';
+import './methods/getImportFileData';
+import './methods/downloadPublicImportFile';
 import './startup/setImportsToInvalid';
+import './startup/store';
 
 export {
 	Base,

packages/rocketchat-importer/server/methods/downloadPublicImportFile.js

@@ -0,0 +1,46 @@
+import { Meteor } from 'meteor/meteor';
+import { Importers } from 'meteor/rocketchat:importer';
+import { RocketChatImportFileInstance } from '../startup/store';
+import { ProgressStep } from '../../lib/ImporterProgressStep';
+import http from 'http';
+
+Meteor.methods({
+	downloadPublicImportFile(fileUrl, importerKey) {
+		const userId = Meteor.userId();
+		console.log(fileUrl);
+
+		if (!userId) {
+			throw new Meteor.Error('error-invalid-user', 'Invalid user', { method: 'downloadPublicImportFile' });
+		}
+
+		if (!RocketChat.authz.hasRole(userId, 'admin')) {
+			throw new Meteor.Error('not_authorized', 'User not authorized', { method: 'downloadPublicImportFile' });
+		}
+
+		const importer = Importers.get(importerKey);
+		if (!importer) {
+			throw new Meteor.Error('error-importer-not-defined', `The importer (${ importerKey }) has no import class defined.`, { method: 'downloadPublicImportFile' });
+		}
+
+		const oldFileName = fileUrl.substring(fileUrl.lastIndexOf('/') + 1);
+		const date = new Date();
+		const dateStr = `${ date.getUTCFullYear() }${ date.getUTCMonth() }${ date.getUTCDate() }${ date.getUTCHours() }${ date.getUTCMinutes() }${ date.getUTCSeconds() }`;
+		const newFileName = `${ dateStr }_${ userId }_${ oldFileName }`;
+
+		importer.instance.startFileUpload(newFileName);
+		importer.instance.updateProgress(ProgressStep.DOWNLOADING_FILE_URL);
+
+		const writeStream = RocketChatImportFileInstance.createWriteStream(newFileName);
+		http.get(fileUrl, function(response) {
+			response.pipe(writeStream);
+		});
+
+		writeStream.on('error', Meteor.bindEnvironment(() => {
+			importer.instance.updateProgress(ProgressStep.ERROR);
+		}));
+
+		writeStream.on('end', Meteor.bindEnvironment(() => {
+			importer.instance.updateProgress(ProgressStep.DOWNLOAD_COMPLETE);
+		}));
+	},
+});

packages/rocketchat-importer/server/methods/getImportFileData.js

@@ -0,0 +1,55 @@
+import { RocketChatImportFileInstance } from '../startup/store';
+import { Meteor } from 'meteor/meteor';
+import { Importers } from 'meteor/rocketchat:importer';
+import { ProgressStep } from '../../lib/ImporterProgressStep';
+import path from 'path';
+
+Meteor.methods({
+	getImportFileData(importerKey) {
+		const userId = Meteor.userId();
+
+		if (!userId) {
+			throw new Meteor.Error('error-invalid-user', 'Invalid user', { method: 'getImportFileData' });
+		}
+
+		if (!RocketChat.authz.hasRole(userId, 'admin')) {
+			throw new Meteor.Error('not_authorized', 'User not authorized', { method: 'getImportFileData' });
+		}
+
+		const importer = Importers.get(importerKey);
+		if (!importer) {
+			throw new Meteor.Error('error-importer-not-defined', `The importer (${ importerKey }) has no import class defined.`, { method: 'getImportFileData' });
+		}
+
+		if (!importer.instance) {
+			return undefined;
+		}
+
+		if (importer.instance.progress.step === ProgressStep.DOWNLOADING_FILE_URL) {
+			return { waiting: true };
+		}
+
+		const fileName = importer.instance.importRecord.file;
+		const fullFilePath = path.join(RocketChatImportFileInstance.absolutePath, fileName);
+		const results = importer.instance.prepareUsingLocalFile(fullFilePath);
+
+		if (results instanceof Promise) {
+			return results.then((data) => {
+				importer.instance.updateRecord({
+					fileData: data,
+				});
+
+				return data;
+			}).catch((e) => {
+				throw new Meteor.Error(e);
+			});
+
+		} else {
+			importer.instance.updateRecord({
+				fileData: results,
+			});
+
+			return results;
+		}
+	},
+});

packages/rocketchat-importer/server/methods/uploadImportFile.js

@@ -0,0 +1,38 @@
+import { Meteor } from 'meteor/meteor';
+import { Importers } from 'meteor/rocketchat:importer';
+import { RocketChatFile } from 'meteor/rocketchat:file';
+import { RocketChatImportFileInstance } from '../startup/store';
+
+Meteor.methods({
+	uploadImportFile(binaryContent, contentType, fileName, importerKey) {
+		const userId = Meteor.userId();
+
+		if (!userId) {
+			throw new Meteor.Error('error-invalid-user', 'Invalid user', { method: 'uploadImportFile' });
+		}
+
+		if (!RocketChat.authz.hasRole(userId, 'admin')) {
+			throw new Meteor.Error('not_authorized', 'User not authorized', { method: 'uploadImportFile' });
+		}
+
+		const importer = Importers.get(importerKey);
+		if (!importer) {
+			throw new Meteor.Error('error-importer-not-defined', `The importer (${ importerKey }) has no import class defined.`, { method: 'uploadImportFile' });
+		}
+
+		const date = new Date();
+		const dateStr = `${ date.getUTCFullYear() }${ date.getUTCMonth() }${ date.getUTCDate() }${ date.getUTCHours() }${ date.getUTCMinutes() }${ date.getUTCSeconds() }`;
+		const newFileName = `${ dateStr }_${ userId }_${ fileName }`;
+
+		importer.instance.startFileUpload(newFileName, contentType);
+
+		const file = new Buffer(binaryContent, 'binary');
+		const readStream = RocketChatFile.bufferToStream(file);
+		const writeStream = RocketChatImportFileInstance.createWriteStream(newFileName, contentType);
+
+		writeStream.on('end', Meteor.bindEnvironment(() => {
+		}));
+
+		readStream.pipe(writeStream);
+	},
+});

packages/rocketchat-importer/server/models/Imports.js

@@ -1,9 +1,32 @@
+import { ProgressStep } from '../../lib/ImporterProgressStep';
 import { RocketChat } from 'meteor/rocketchat:lib';
 
 class ImportsModel extends RocketChat.models._Base {
 	constructor() {
 		super('import');
 	}
+
+	findPendingImport(key) {
+		// Finds the latest import operation
+		const data = this.findOne({ importerKey: key }, { createdAt : -1 });
+		if (!data || !data.status) {
+			return data;
+		}
+
+		// But only returns it if it is still pending
+
+		const forbiddenStatus = [
+			ProgressStep.DONE,
+			ProgressStep.ERROR,
+			ProgressStep.CANCELLED,
+		];
+
+		if (forbiddenStatus.indexOf(data.status) >= 0) {
+			return undefined;
+		}
+
+		return data;
+	}
 }
 
 export const Imports = new ImportsModel();

packages/rocketchat-importer/server/startup/store.js

@@ -0,0 +1,20 @@
+import { Meteor } from 'meteor/meteor';
+import { RocketChatFile } from 'meteor/rocketchat:file';
+
+export let RocketChatImportFileInstance;
+
+Meteor.startup(function() {
+	const RocketChatStore = RocketChatFile.FileSystem;
+
+	let path = '~/uploads';
+	if (RocketChat.settings.get('ImportFile_FileSystemPath') != null) {
+		if (RocketChat.settings.get('ImportFile_FileSystemPath').trim() !== '') {
+			path = RocketChat.settings.get('ImportFile_FileSystemPath');
+		}
+	}
+
+	RocketChatImportFileInstance = new RocketChatStore({
+		name: 'import_files',
+		absolutePath: path,
+	});
+});