apitech
/
Rocket.Chat
mirror of https://github.com/RocketChat/Rocket.Chat
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #7456 from seekingalpha/csv-import

Browse Source 
[FIX] Csv importer: work with more problematic data
pull/7677/head^2
Rodrigo Nascimento 8 years ago committed by GitHub
parent c56f09b08f 1440fe38d3
commit 76b22c2962
9 changed files with 114 additions and 39 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json
  2. 2
      packages/rocketchat-importer-csv/package.js
  3. 61
      packages/rocketchat-importer-csv/server.js
  4. 6
      packages/rocketchat-importer-hipchat-enterprise/server.js
  5. 6
      packages/rocketchat-importer-hipchat/server.js
  6. 6
      packages/rocketchat-importer-slack/server.js
  7. 56
      packages/rocketchat-importer/client/admin/adminImportPrepare.html
  8. 6
      packages/rocketchat-importer/client/admin/adminImportPrepare.js
  9. 4
      packages/rocketchat-importer/server/classes/ImporterSelection.js

6
packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json generated
Unescape View File

@ -1,9 +1,9 @@
{
"dependencies": {
"csv-parse": {
"version": "1.1.7",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.1.7.tgz",
"from": "csv-parse@1.1.7"
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.2.0.tgz",
"from": "csv-parse@1.2.0"
}
}
}

2
packages/rocketchat-importer-csv/package.js
Unescape View File

@ -17,5 +17,5 @@ Package.onUse(function(api) {
});
Npm.depends({
'csv-parse': '1.1.7'
'csv-parse': '1.2.0'
});

61
packages/rocketchat-importer-csv/server.js
Unescape View File

@ -131,9 +131,10 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = tempChannels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;
super.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
startImport(importSelection) {
@ -232,16 +233,60 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
}
this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }});
//If no channels file, collect channel map from DB for message-only import
if (this.channels.channels.length === 0) {
for (const cname of this.messages.keys()) {
Meteor.runAsUser(startedByUserId, () => {
const existantRoom = RocketChat.models.Rooms.findOneByName(cname);
if (existantRoom || cname.toUpperCase() === 'GENERAL') {
this.channels.channels.push({
id: cname.replace('.', '_'),
name: cname,
rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id),
do_import: true
});
}
});
}
}
//If no users file, collect user map from DB for message-only import
if (this.users.users.length === 0) {
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel || !csvChannel.do_import) {
continue;
}
Meteor.runAsUser(startedByUserId, () => {
for (const msgs of messagesMap.values()) {
for (const msg of msgs.messages) {
if (!this.getUserFromUsername(msg.username)) {
const user = RocketChat.models.Users.findOneByUsername(msg.username);
if (user) {
this.users.users.push({
rocketId: user._id,
username: user.username
});
}
}
}
}
});
}
}
//Import the Messages
super.updateProgress(Importer.ProgressStep.IMPORTING_MESSAGES);
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel.do_import) {
if (!csvChannel || !csvChannel.do_import) {
continue;
}
const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } });
Meteor.runAsUser(startedByUserId, () => {
const timestamps = {};
for (const [msgGroupData, msgs] of messagesMap.entries()) {
super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` });
for (const msg of msgs.messages) {
@ -253,8 +298,15 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
const creator = this.getUserFromUsername(msg.username);
if (creator) {
let suffix = '';
if (timestamps[msg.ts] === undefined) {
timestamps[msg.ts] = 1;
} else {
suffix = `-${ timestamps[msg.ts] }`;
timestamps[msg.ts] += 1;
}
const msgObj = {
_id: `csv-${ csvChannel.id }-${ msg.ts }`,
_id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`,
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
rid: room._id,
@ -285,8 +337,9 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
getChannelFromName(channelName) {

6
packages/rocketchat-importer-hipchat-enterprise/server.js
Unescape View File

@ -190,10 +190,11 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba
const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true));
const selectionChannels = tempRooms.map((r) => new Importer.SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate));
const selectionMessages = this.importRecord.count.messages;
super.updateProgress(Importer.ProgressStep.USER_SELECTION);
resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels));
resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages));
}));
//Wish I could make this cleaner :(
@ -431,8 +432,9 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
getChannelFromRoomIdentifier(roomIdentifier) {

6
packages/rocketchat-importer-hipchat/server.js
Unescape View File

@ -131,8 +131,9 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = tempRooms.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
startImport(importSelection) {
@ -331,7 +332,8 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = this.channels.channels.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
}

6
packages/rocketchat-importer-slack/server.js
Unescape View File

@ -92,8 +92,9 @@ Importer.Slack = class extends Importer.Base {
}
const selectionUsers = tempUsers.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = tempChannels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
startImport(importSelection) {
super.startImport(importSelection);
@ -431,6 +432,7 @@ Importer.Slack = class extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = this.channels.channels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
};

56
packages/rocketchat-importer/client/admin/adminImportPrepare.html
Unescape View File

@ -33,35 +33,43 @@
</div>
</div>
<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
{{#if users.length}}
<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
</div>
</div>
{{/if}}
{{#if channels.length}}
<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
{{/each}}
</ul>
</div>
</div>
</div>
{{/if}}
<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/each}}
</ul>
</div>
<h1>{{_ "Messages"}}: {{message_count}}</h1>
</div>
{{else}}
{{#if isPreparing}}

6
packages/rocketchat-importer/client/admin/adminImportPrepare.js
Unescape View File

@ -27,6 +27,9 @@ Template.adminImportPrepare.helpers({
},
channels() {
return Template.instance().channels.get();
},
message_count() {
return Template.instance().message_count.get();
}
});
@ -70,6 +73,7 @@ Template.adminImportPrepare.events({
template.users.set(data.users);
template.channels.set(data.channels);
template.message_count.set(data.message_count);
template.loaded.set(true);
template.preparing.set(false);
});
@ -131,6 +135,7 @@ Template.adminImportPrepare.onCreated(function() {
this.loaded = new ReactiveVar(false);
this.users = new ReactiveVar([]);
this.channels = new ReactiveVar([]);
this.message_count = new ReactiveVar(0);
function loadSelection(progress) {
if ((progress != null ? progress.step : undefined)) {
@ -146,6 +151,7 @@ Template.adminImportPrepare.onCreated(function() {
}
instance.users.set(data.users);
instance.channels.set(data.channels);
instance.message_count.set(data.message_count);
instance.loaded.set(true);
return instance.preparing.set(false);
});

4
packages/rocketchat-importer/server/classes/ImporterSelection.js
Unescape View File

@ -6,10 +6,12 @@ Importer.Selection = (Importer.Selection = class Selection {
// @param [String] name the name of the Importer
// @param [Array<Importer.User>] users the array of users
// @param [Array<Importer.Channel>] channels the array of channels
// @param [Integer] number of collected messages
//
constructor(name, users, channels) {
constructor(name, users, channels, message_count) {
this.name = name;
this.users = users;
this.channels = channels;
this.message_count = message_count;
}
});

Write Preview
Loading…
Cancel
Save