import { Meteor } from 'meteor/meteor'; import { Accounts } from 'meteor/accounts-base'; import { Random } from 'meteor/random'; import { Base, ProgressStep, Selection, SelectionChannel, SelectionUser, } from 'meteor/rocketchat:importer'; import { Readable } from 'stream'; import path from 'path'; import s from 'underscore.string'; import TurndownService from 'turndown'; const turndownService = new TurndownService({ strongDelimiter: '*', hr: '', br: '\n', }); turndownService.addRule('strikethrough', { filter: 'img', replacement(content, node) { const src = node.getAttribute('src') || ''; const alt = node.alt || node.title || src; return src ? `[${ alt }](${ src })` : ''; }, }); export class HipChatEnterpriseImporter extends Base { constructor(info) { super(info); this.Readable = Readable; this.zlib = require('zlib'); this.tarStream = require('tar-stream'); this.extract = this.tarStream.extract(); this.path = path; this.messages = new Map(); this.directMessages = new Map(); } prepare(dataURI, sentContentType, fileName) { super.prepare(dataURI, sentContentType, fileName); const tempUsers = []; const tempRooms = []; const tempMessages = new Map(); const tempDirectMessages = new Map(); const promise = new Promise((resolve, reject) => { this.extract.on('entry', Meteor.bindEnvironment((header, stream, next) => { if (!header.name.endsWith('.json')) { stream.resume(); return next(); } const info = this.path.parse(header.name); const data = []; stream.on('data', Meteor.bindEnvironment((chunk) => { data.push(chunk); })); stream.on('end', Meteor.bindEnvironment(() => { this.logger.debug(`Processing the file: ${ header.name }`); const dataString = Buffer.concat(data).toString(); const file = JSON.parse(dataString); if (info.base === 'users.json') { super.updateProgress(ProgressStep.PREPARING_USERS); for (const u of file) { // if (!u.User.email) { // // continue; // } tempUsers.push({ id: u.User.id, email: u.User.email, name: u.User.name, username: u.User.mention_name, avatar: u.User.avatar && u.User.avatar.replace(/\n/g, ''), timezone: u.User.timezone, isDeleted: u.User.is_deleted, }); } } else if (info.base === 'rooms.json') { super.updateProgress(ProgressStep.PREPARING_CHANNELS); for (const r of file) { tempRooms.push({ id: r.Room.id, creator: r.Room.owner, created: new Date(r.Room.created), name: s.slugify(r.Room.name), isPrivate: r.Room.privacy === 'private', isArchived: r.Room.is_archived, topic: r.Room.topic, }); } } else if (info.base === 'history.json') { const [type, id] = info.dir.split('/'); // ['users', '1'] const roomIdentifier = `${ type }/${ id }`; if (type === 'users') { const msgs = []; for (const m of file) { if (m.PrivateUserMessage) { msgs.push({ type: 'user', id: `hipchatenterprise-${ m.PrivateUserMessage.id }`, senderId: m.PrivateUserMessage.sender.id, receiverId: m.PrivateUserMessage.receiver.id, text: m.PrivateUserMessage.message.indexOf('/me ') === -1 ? m.PrivateUserMessage.message : `${ m.PrivateUserMessage.message.replace(/\/me /, '_') }_`, ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]), attachment: m.PrivateUserMessage.attachment, attachment_path: m.PrivateUserMessage.attachment_path, }); } } tempDirectMessages.set(roomIdentifier, msgs); } else if (type === 'rooms') { const roomMsgs = []; for (const m of file) { if (m.UserMessage) { roomMsgs.push({ type: 'user', id: `hipchatenterprise-${ id }-${ m.UserMessage.id }`, userId: m.UserMessage.sender.id, text: m.UserMessage.message.indexOf('/me ') === -1 ? m.UserMessage.message : `${ m.UserMessage.message.replace(/\/me /, '_') }_`, ts: new Date(m.UserMessage.timestamp.split(' ')[0]), }); } else if (m.NotificationMessage) { const text = m.NotificationMessage.message.indexOf('/me ') === -1 ? m.NotificationMessage.message : `${ m.NotificationMessage.message.replace(/\/me /, '_') }_`; roomMsgs.push({ type: 'user', id: `hipchatenterprise-${ id }-${ m.NotificationMessage.id }`, userId: 'rocket.cat', alias: m.NotificationMessage.sender, text: m.NotificationMessage.message_format === 'html' ? turndownService.turndown(text) : text, ts: new Date(m.NotificationMessage.timestamp.split(' ')[0]), }); } else if (m.TopicRoomMessage) { roomMsgs.push({ type: 'topic', id: `hipchatenterprise-${ id }-${ m.TopicRoomMessage.id }`, userId: m.TopicRoomMessage.sender.id, ts: new Date(m.TopicRoomMessage.timestamp.split(' ')[0]), text: m.TopicRoomMessage.message, }); } else { this.logger.warn('HipChat Enterprise importer isn\'t configured to handle this message:', m); } } tempMessages.set(roomIdentifier, roomMsgs); } else { this.logger.warn(`HipChat Enterprise importer isn't configured to handle "${ type }" files.`); } } else { // What are these files!? this.logger.warn(`HipChat Enterprise importer doesn't know what to do with the file "${ header.name }" :o`, info); } next(); })); stream.on('error', () => next()); stream.resume(); })); this.extract.on('error', (err) => { this.logger.warn('extract error:', err); reject(); }); this.extract.on('finish', Meteor.bindEnvironment(() => { // Insert the users record, eventually this might have to be split into several ones as well // if someone tries to import a several thousands users instance const usersId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'users', users: tempUsers }); this.users = this.collection.findOne(usersId); super.updateRecord({ 'count.users': tempUsers.length }); super.addCountToTotal(tempUsers.length); // Insert the channels records. const channelsId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'channels', channels: tempRooms }); this.channels = this.collection.findOne(channelsId); super.updateRecord({ 'count.channels': tempRooms.length }); super.addCountToTotal(tempRooms.length); // Save the messages records to the import record for `startImport` usage super.updateProgress(ProgressStep.PREPARING_MESSAGES); let messagesCount = 0; for (const [channel, msgs] of tempMessages.entries()) { if (!this.messages.get(channel)) { this.messages.set(channel, new Map()); } messagesCount += msgs.length; super.updateRecord({ messagesstatus: channel }); if (Base.getBSONSize(msgs) > Base.getMaxBSONSize()) { Base.getBSONSafeArraysFromAnArray(msgs).forEach((splitMsg, i) => { const messagesId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }/${ i }`, messages: splitMsg }); this.messages.get(channel).set(`${ channel }.${ i }`, this.collection.findOne(messagesId)); }); } else { const messagesId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'messages', name: `${ channel }`, messages: msgs }); this.messages.get(channel).set(channel, this.collection.findOne(messagesId)); } } for (const [directMsgUser, msgs] of tempDirectMessages.entries()) { this.logger.debug(`Preparing the direct messages for: ${ directMsgUser }`); if (!this.directMessages.get(directMsgUser)) { this.directMessages.set(directMsgUser, new Map()); } messagesCount += msgs.length; super.updateRecord({ messagesstatus: directMsgUser }); if (Base.getBSONSize(msgs) > Base.getMaxBSONSize()) { Base.getBSONSafeArraysFromAnArray(msgs).forEach((splitMsg, i) => { const messagesId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'directMessages', name: `${ directMsgUser }/${ i }`, messages: splitMsg }); this.directMessages.get(directMsgUser).set(`${ directMsgUser }.${ i }`, this.collection.findOne(messagesId)); }); } else { const messagesId = this.collection.insert({ import: this.importRecord._id, importer: this.name, type: 'directMessages', name: `${ directMsgUser }`, messages: msgs }); this.directMessages.get(directMsgUser).set(directMsgUser, this.collection.findOne(messagesId)); } } super.updateRecord({ 'count.messages': messagesCount, messagesstatus: null }); super.addCountToTotal(messagesCount); // Ensure we have some users, channels, and messages if (tempUsers.length === 0 || tempRooms.length === 0 || messagesCount === 0) { this.logger.warn(`The loaded users count ${ tempUsers.length }, the loaded rooms ${ tempRooms.length }, and the loaded messages ${ messagesCount }`); super.updateProgress(ProgressStep.ERROR); reject(); return; } const selectionUsers = tempUsers.map((u) => new SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true)); const selectionChannels = tempRooms.map((r) => new SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate)); const selectionMessages = this.importRecord.count.messages; super.updateProgress(ProgressStep.USER_SELECTION); resolve(new Selection(this.name, selectionUsers, selectionChannels, selectionMessages)); })); // Wish I could make this cleaner :( const split = dataURI.split(','); const read = new this.Readable; read.push(new Buffer(split[split.length - 1], 'base64')); read.push(null); read.pipe(this.zlib.createGunzip()).pipe(this.extract); }); return promise; } startImport(importSelection) { super.startImport(importSelection); const started = Date.now(); // Ensure we're only going to import the users that the user has selected for (const user of importSelection.users) { for (const u of this.users.users) { if (u.id === user.user_id) { u.do_import = user.do_import; } } } this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); // Ensure we're only importing the channels the user has selected. for (const channel of importSelection.channels) { for (const c of this.channels.channels) { if (c.id === channel.channel_id) { c.do_import = channel.do_import; } } } this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); const startedByUserId = Meteor.userId(); Meteor.defer(() => { super.updateProgress(ProgressStep.IMPORTING_USERS); try { // Import the users for (const u of this.users.users) { this.logger.debug(`Starting the user import: ${ u.username } and are we importing them? ${ u.do_import }`); if (!u.do_import) { continue; } Meteor.runAsUser(startedByUserId, () => { let existantUser; if (u.email) { RocketChat.models.Users.findOneByEmailAddress(u.email); } // If we couldn't find one by their email address, try to find an existing user by their username if (!existantUser) { existantUser = RocketChat.models.Users.findOneByUsername(u.username); } if (existantUser) { // since we have an existing user, let's try a few things u.rocketId = existantUser._id; RocketChat.models.Users.update({ _id: u.rocketId }, { $addToSet: { importIds: u.id } }); } else { const user = { email: u.email, password: Random.id() }; if (!user.email) { delete user.email; user.username = u.username; } const userId = Accounts.createUser(user); Meteor.runAsUser(userId, () => { Meteor.call('setUsername', u.username, { joinDefaultChannelsSilenced: true }); // TODO: Use moment timezone to calc the time offset - Meteor.call 'userSetUtcOffset', user.tz_offset / 3600 RocketChat.models.Users.setName(userId, u.name); // TODO: Think about using a custom field for the users "title" field if (u.avatar) { Meteor.call('setAvatarFromService', `data:image/png;base64,${ u.avatar }`); } // Deleted users are 'inactive' users in Rocket.Chat if (u.deleted) { Meteor.call('setUserActiveStatus', userId, false); } RocketChat.models.Users.update({ _id: userId }, { $addToSet: { importIds: u.id } }); u.rocketId = userId; }); } super.addCountCompleted(1); }); } this.collection.update({ _id: this.users._id }, { $set: { users: this.users.users } }); // Import the channels super.updateProgress(ProgressStep.IMPORTING_CHANNELS); for (const c of this.channels.channels) { if (!c.do_import) { continue; } Meteor.runAsUser(startedByUserId, () => { const existantRoom = RocketChat.models.Rooms.findOneByName(c.name); // If the room exists or the name of it is 'general', then we don't need to create it again if (existantRoom || c.name.toUpperCase() === 'GENERAL') { c.rocketId = c.name.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id; RocketChat.models.Rooms.update({ _id: c.rocketId }, { $addToSet: { importIds: c.id } }); } else { // Find the rocketchatId of the user who created this channel let creatorId = startedByUserId; for (const u of this.users.users) { if (u.id === c.creator && u.do_import) { creatorId = u.rocketId; } } // Create the channel Meteor.runAsUser(creatorId, () => { const roomInfo = Meteor.call(c.isPrivate ? 'createPrivateGroup' : 'createChannel', c.name, []); c.rocketId = roomInfo.rid; }); RocketChat.models.Rooms.update({ _id: c.rocketId }, { $set: { ts: c.created, topic: c.topic }, $addToSet: { importIds: c.id } }); } super.addCountCompleted(1); }); } this.collection.update({ _id: this.channels._id }, { $set: { channels: this.channels.channels } }); // Import the Messages super.updateProgress(ProgressStep.IMPORTING_MESSAGES); for (const [ch, messagesMap] of this.messages.entries()) { const hipChannel = this.getChannelFromRoomIdentifier(ch); if (!hipChannel.do_import) { continue; } const room = RocketChat.models.Rooms.findOneById(hipChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); Meteor.runAsUser(startedByUserId, () => { for (const [msgGroupData, msgs] of messagesMap.entries()) { super.updateRecord({ messagesstatus: `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { if (isNaN(msg.ts)) { this.logger.warn(`Timestamp on a message in ${ ch }/${ msgGroupData } is invalid`); super.addCountCompleted(1); continue; } const creator = this.getRocketUserFromUserId(msg.userId); if (creator) { switch (msg.type) { case 'user': RocketChat.sendMessage(creator, { _id: msg.id, ts: msg.ts, msg: msg.text, rid: room._id, alias: msg.alias, u: { _id: creator._id, username: creator.username, }, }, room, true); break; case 'topic': RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser('room_changed_topic', room._id, msg.text, creator, { _id: msg.id, ts: msg.ts }); break; } } super.addCountCompleted(1); } } }); } // Import the Direct Messages for (const [directMsgRoom, directMessagesMap] of this.directMessages.entries()) { const hipUser = this.getUserFromDirectMessageIdentifier(directMsgRoom); if (!hipUser || !hipUser.do_import) { continue; } // Verify this direct message user's room is valid (confusing but idk how else to explain it) if (!this.getRocketUserFromUserId(hipUser.id)) { continue; } for (const [msgGroupData, msgs] of directMessagesMap.entries()) { super.updateRecord({ messagesstatus: `${ directMsgRoom }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { if (isNaN(msg.ts)) { this.logger.warn(`Timestamp on a message in ${ directMsgRoom }/${ msgGroupData } is invalid`); super.addCountCompleted(1); continue; } // make sure the message sender is a valid user inside rocket.chat const sender = this.getRocketUserFromUserId(msg.senderId); if (!sender) { continue; } // make sure the receiver of the message is a valid rocket.chat user const receiver = this.getRocketUserFromUserId(msg.receiverId); if (!receiver) { continue; } let room = RocketChat.models.Rooms.findOneById([receiver._id, sender._id].sort().join('')); if (!room) { Meteor.runAsUser(sender._id, () => { const roomInfo = Meteor.call('createDirectMessage', receiver.username); room = RocketChat.models.Rooms.findOneById(roomInfo.rid); }); } Meteor.runAsUser(sender._id, () => { if (msg.attachment_path) { const details = { message_id: msg.id, name: msg.attachment.name, size: msg.attachment.size, userId: sender._id, rid: room._id, }; this.uploadFile(details, msg.attachment.url, sender, room, msg.ts); } else { RocketChat.sendMessage(sender, { _id: msg.id, ts: msg.ts, msg: msg.text, rid: room._id, u: { _id: sender._id, username: sender.username, }, }, room, true); } }); } } } super.updateProgress(ProgressStep.FINISHING); super.updateProgress(ProgressStep.DONE); } catch (e) { super.updateRecord({ 'error-record': JSON.stringify(e, Object.getOwnPropertyNames(e)) }); this.logger.error(e); super.updateProgress(ProgressStep.ERROR); } const timeTook = Date.now() - started; this.logger.log(`HipChat Enterprise Import took ${ timeTook } milliseconds.`); }); return super.getProgress(); } getSelection() { const selectionUsers = this.users.users.map((u) => new SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = this.channels.channels.map((c) => new SelectionChannel(c.id, c.name, false, true, c.isPrivate)); const selectionMessages = this.importRecord.count.messages; return new Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } getChannelFromRoomIdentifier(roomIdentifier) { for (const ch of this.channels.channels) { if (`rooms/${ ch.id }` === roomIdentifier) { return ch; } } } getUserFromDirectMessageIdentifier(directIdentifier) { for (const u of this.users.users) { if (`users/${ u.id }` === directIdentifier) { return u; } } } getRocketUserFromUserId(userId) { if (userId === 'rocket.cat') { return RocketChat.models.Users.findOneById(userId, { fields: { username: 1 } }); } for (const u of this.users.users) { if (u.id === userId) { return RocketChat.models.Users.findOneById(u.rocketId, { fields: { username: 1 } }); } } } }