[NEW] GDPR - Right to access and Data Portability (#9906)

* Implemented basic JSON generation

* Added new data to message json

* Changed export operation structure

* Changed code to use forEach instead of fetch

* Split the operation into two independent commands

* File download, zip generation, admin settings

* Use syncedcron to process data downloads

* Added download URL

* Sending emails when the download file is ready

* Allow usage of GridFS as storage for the finished file.

* Lint

* Added support for Google and Amazon as storage types

* Split the options to download and export data

* Removed commented code
pull/10535/head
Pierre H. Lehnen 7 years ago committed by Rodrigo Nascimento
parent 816d110acb
commit fee30ad6f9
  1. 1
      .meteor/packages
  2. 1
      .meteor/versions
  3. 1
      package.json
  4. 1
      packages/rocketchat-file-upload/client/lib/fileUploadHandler.js
  5. 16
      packages/rocketchat-file-upload/lib/FileUploadBase.js
  6. 25
      packages/rocketchat-file-upload/server/config/AmazonS3.js
  7. 42
      packages/rocketchat-file-upload/server/config/FileSystem.js
  8. 29
      packages/rocketchat-file-upload/server/config/GoogleStorage.js
  9. 40
      packages/rocketchat-file-upload/server/config/GridFS.js
  10. 1
      packages/rocketchat-file-upload/server/config/_configUploadStorage.js
  11. 37
      packages/rocketchat-file-upload/server/lib/FileUpload.js
  12. 14
      packages/rocketchat-i18n/i18n/en.i18n.json
  13. 6
      packages/rocketchat-lib/client/models/UserDataFiles.js
  14. 2
      packages/rocketchat-lib/package.js
  15. 78
      packages/rocketchat-lib/server/models/ExportOperations.js
  16. 8
      packages/rocketchat-lib/server/models/Messages.js
  17. 40
      packages/rocketchat-lib/server/models/UserDataFiles.js
  18. 5
      packages/rocketchat-lib/server/models/Users.js
  19. 13
      packages/rocketchat-ui-account/client/accountPreferences.html
  20. 60
      packages/rocketchat-ui-account/client/accountPreferences.js
  21. 18
      packages/rocketchat-user-data-download/package.js
  22. 476
      packages/rocketchat-user-data-download/server/cronProcessDownloads.js
  23. 34
      packages/rocketchat-user-data-download/server/startup/settings.js
  24. 63
      server/methods/requestDataDownload.js
  25. 20
      server/startup/migrations/v110.js

@ -146,6 +146,7 @@ rocketchat:ui-master
rocketchat:ui-message
rocketchat:ui-sidenav
rocketchat:ui-vrecord
rocketchat:user-data-download
rocketchat:version
rocketchat:videobridge
rocketchat:webrtc

@ -237,6 +237,7 @@ rocketchat:ui-master@0.1.0
rocketchat:ui-message@0.1.0
rocketchat:ui-sidenav@0.1.0
rocketchat:ui-vrecord@0.0.1
rocketchat:user-data-download@1.0.0
rocketchat:version@1.0.0
rocketchat:version-check@0.0.1
rocketchat:videobridge@0.2.0

@ -114,6 +114,7 @@
"@google-cloud/storage": "^1.6.0",
"@google-cloud/vision": "^0.15.2",
"adm-zip": "^0.4.7",
"archiver": "^2.1.1",
"atlassian-crowd": "^0.5.0",
"autolinker": "^1.6.2",
"aws-sdk": "^2.199.0",

@ -17,7 +17,6 @@ new UploadFS.Store({
})
});
fileUploadHandler = (directive, meta, file) => {
const store = UploadFS.getStore(directive);

@ -4,7 +4,21 @@ import _ from 'underscore';
UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({
insert(userId, doc) {
return userId || (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0); // allow inserts from slackbridge (message_id = slack-timestamp-milli)
if (userId) {
return true;
}
// allow inserts from slackbridge (message_id = slack-timestamp-milli)
if (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0) {
return true;
}
// allow inserts to the UserDataFiles store
if (doc && doc.store && doc.store.split(':').pop() === 'UserDataFiles') {
return true;
}
return false;
},
update(userId, doc) {
return RocketChat.authz.hasPermission(Meteor.userId(), 'delete-message', doc.rid) || (RocketChat.settings.get('Message_AllowDeleting') && userId === doc.userId);

@ -25,15 +25,35 @@ const get = function(file, req, res) {
}
};
const copy = function(file, out) {
const fileUrl = this.store.getRedirectURL(file);
if (fileUrl) {
const request = /^https:/.test(fileUrl) ? https : http;
request.get(fileUrl, fileRes => fileRes.pipe(out));
} else {
out.end();
}
};
const AmazonS3Uploads = new FileUploadClass({
name: 'AmazonS3:Uploads',
get
get,
copy
// store setted bellow
});
const AmazonS3Avatars = new FileUploadClass({
name: 'AmazonS3:Avatars',
get
get,
copy
// store setted bellow
});
const AmazonS3UserDataFiles = new FileUploadClass({
name: 'AmazonS3:UserDataFiles',
get,
copy
// store setted bellow
});
@ -74,6 +94,7 @@ const configure = _.debounce(function() {
AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config);
AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config);
AmazonS3UserDataFiles.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3UserDataFiles.name, config);
}, 500);
RocketChat.settings.get(/^FileUpload_S3_/, configure);

@ -28,6 +28,22 @@ const FileSystemUploads = new FileUploadClass({
res.end();
return;
}
},
copy(file, out) {
const filePath = this.store.getFilePath(file._id, file);
try {
const stat = Meteor.wrapAsync(fs.stat)(filePath);
if (stat && stat.isFile()) {
file = FileUpload.addExtensionTo(file);
this.store.getReadStream(file._id, file).pipe(out);
}
} catch (e) {
out.end();
return;
}
}
});
@ -54,6 +70,31 @@ const FileSystemAvatars = new FileUploadClass({
}
});
const FileSystemUserDataFiles = new FileUploadClass({
name: 'FileSystem:UserDataFiles',
get(file, req, res) {
const filePath = this.store.getFilePath(file._id, file);
try {
const stat = Meteor.wrapAsync(fs.stat)(filePath);
if (stat && stat.isFile()) {
file = FileUpload.addExtensionTo(file);
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
res.setHeader('Content-Type', file.type);
res.setHeader('Content-Length', file.size);
this.store.getReadStream(file._id, file).pipe(res);
}
} catch (e) {
res.writeHead(404);
res.end();
return;
}
}
});
const createFileSystemStore = _.debounce(function() {
const options = {
@ -62,6 +103,7 @@ const createFileSystemStore = _.debounce(function() {
FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options);
FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options);
FileSystemUserDataFiles.store = FileUpload.configureUploadsStore('Local', FileSystemUserDataFiles.name, options);
// DEPRECATED backwards compatibililty (remove)
UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name];

@ -29,15 +29,39 @@ const get = function(file, req, res) {
});
};
const copy = function(file, out) {
this.store.getRedirectURL(file, (err, fileUrl) => {
if (err) {
console.error(err);
}
if (fileUrl) {
const request = /^https:/.test(fileUrl) ? https : http;
request.get(fileUrl, fileRes => fileRes.pipe(out));
} else {
out.end();
}
});
};
const GoogleCloudStorageUploads = new FileUploadClass({
name: 'GoogleCloudStorage:Uploads',
get
get,
copy
// store setted bellow
});
const GoogleCloudStorageAvatars = new FileUploadClass({
name: 'GoogleCloudStorage:Avatars',
get
get,
copy
// store setted bellow
});
const GoogleCloudStorageUserDataFiles = new FileUploadClass({
name: 'GoogleCloudStorage:UserDataFiles',
get,
copy
// store setted bellow
});
@ -64,6 +88,7 @@ const configure = _.debounce(function() {
GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config);
GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config);
GoogleCloudStorageUserDataFiles.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUserDataFiles.name, config);
}, 500);
RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure);

@ -62,7 +62,6 @@ const getByteRange = function(header) {
return null;
};
// code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310
const readFromGridFS = function(storeName, fileId, file, req, res) {
const store = UploadFS.getStore(storeName);
@ -123,10 +122,26 @@ const readFromGridFS = function(storeName, fileId, file, req, res) {
}
};
const copyFromGridFS = function(storeName, fileId, file, out) {
const store = UploadFS.getStore(storeName);
const rs = store.getReadStream(fileId, file);
[rs, out].forEach(stream => stream.on('error', function(err) {
store.onReadError.call(store, err, fileId, file);
out.end();
}));
rs.pipe(out);
};
FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', {
collectionName: 'rocketchat_uploads'
});
FileUpload.configureUploadsStore('GridFS', 'GridFS:UserDataFiles', {
collectionName: 'rocketchat_userDataFiles'
});
// DEPRECATED: backwards compatibility (remove)
UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads'];
@ -147,6 +162,29 @@ new FileUploadClass({
res.setHeader('Content-Length', file.size);
return readFromGridFS(file.store, file._id, file, req, res);
},
copy(file, out) {
copyFromGridFS(file.store, file._id, file, out);
}
});
new FileUploadClass({
name: 'GridFS:UserDataFiles',
get(file, req, res) {
file = FileUpload.addExtensionTo(file);
res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
res.setHeader('Content-Type', file.type);
res.setHeader('Content-Length', file.size);
return readFromGridFS(file.store, file._id, file, req, res);
},
copy(file, out) {
copyFromGridFS(file.store, file._id, file, out);
}
});

@ -14,6 +14,7 @@ const configStore = _.debounce(() => {
console.log('Setting default file store to', store);
UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`);
UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`);
UploadFS.getStores().UserDataFiles = UploadFS.getStore(`${ store }:UserDataFiles`);
}
}, 1000);

@ -58,6 +58,25 @@ Object.assign(FileUpload, {
};
},
defaultUserDataFiles() {
return {
collection: RocketChat.models.UserDataFiles.model,
getPath(file) {
return `${ RocketChat.settings.get('uniqueID') }/uploads/userData/${ file.userId }`;
},
onValidate: FileUpload.uploadsOnValidate,
onRead(fileId, file, req, res) {
if (!FileUpload.requestCanAccessFiles(req)) {
res.writeHead(403);
return false;
}
res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`);
return true;
}
};
},
avatarsOnValidate(file) {
if (RocketChat.settings.get('Accounts_AvatarResize') !== true) {
return;
@ -229,16 +248,30 @@ Object.assign(FileUpload, {
}
res.writeHead(404);
res.end();
},
copy(file, targetFile) {
const store = this.getStoreByName(file.store);
const out = fs.createWriteStream(targetFile);
file = FileUpload.addExtensionTo(file);
if (store.copy) {
store.copy(file, out);
return true;
}
return false;
}
});
export class FileUploadClass {
constructor({ name, model, store, get, insert, getStore }) {
constructor({ name, model, store, get, insert, getStore, copy }) {
this.name = name;
this.model = model || this.getModelFromName();
this._store = store || UploadFS.getStore(name);
this.get = get;
this.copy = copy;
if (insert) {
this.insert = insert;

@ -627,6 +627,7 @@
"Domain_removed": "Domain Removed",
"Domains": "Domains",
"Domains_allowed_to_embed_the_livechat_widget": "Comma-separated list of domains allowed to embed the livechat widget. Leave blank to allow all domains.",
"Download_My_Data" : "Download My Data",
"Download_Snippet": "Download",
"Drop_to_upload_file": "Drop to upload file",
"Dry_run": "Dry run",
@ -784,6 +785,7 @@
"Example_s": "Example: <code class=\"inline\">%s</code>",
"Exclude_Botnames": "Exclude Bots",
"Exclude_Botnames_Description": "Do not propagate messages from bots whose name matches the regular expression above. If left empty, all messages from bots will be propagated.",
"Export_My_Data" : "Export My Data",
"External_Service": "External Service",
"External_Queue_Service_URL": "External Queue Service URL",
"Facebook_Page": "Facebook Page",
@ -2178,6 +2180,18 @@
"User_uploaded_file": "Uploaded a file",
"User_uploaded_image": "Uploaded an image",
"User_Presence": "User Presence",
"UserDataDownload" : "User Data Download",
"UserData_EnableDownload" : "Enable User Data Download",
"UserData_FileSystemPath" : "System Path (Exported Files)",
"UserData_FileSystemZipPath" : "System Path (Compressed File)",
"UserData_ProcessingFrequency" : "Processing Frequency (Minutes)",
"UserData_MessageLimitPerRequest" : "Message Limit per Request",
"UserDataDownload_EmailSubject" : "Your Data File is Ready to Download",
"UserDataDownload_EmailBody" : "Your data file is now ready to download. Click <a href=\"__download_link__\">here</a> to download it.",
"UserDataDownload_Requested" : "Download File Requested",
"UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.",
"UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.",
"UserDataDownload_CompletedRequestExisted_Text" : "Your data file was already generated. Check your email account for the download link.",
"Username": "Username",
"Username_and_message_must_not_be_empty": "Username and message must not be empty.",
"Username_cant_be_empty": "The username cannot be empty",

@ -0,0 +1,6 @@
RocketChat.models.UserDataFiles = new class extends RocketChat.models._Base {
constructor() {
super();
this._initModel('userDataFiles');
}
};

@ -126,6 +126,8 @@ Package.onUse(function(api) {
api.addFiles('server/models/Subscriptions.js', 'server');
api.addFiles('server/models/Uploads.js', 'server');
api.addFiles('server/models/Users.js', 'server');
api.addFiles('server/models/ExportOperations.js', 'server');
api.addFiles('server/models/UserDataFiles.js', 'server');
api.addFiles('server/oauth/oauth.js', 'server');
api.addFiles('server/oauth/facebook.js', 'server');

@ -0,0 +1,78 @@
import _ from 'underscore';
RocketChat.models.ExportOperations = new class ModelExportOperations extends RocketChat.models._Base {
constructor() {
super('export_operations');
this.tryEnsureIndex({ 'userId': 1 });
this.tryEnsureIndex({ 'status': 1 });
}
// FIND
findById(id) {
const query = {_id: id};
return this.find(query);
}
findLastOperationByUser(userId, fullExport = false, options = {}) {
const query = {
userId,
fullExport
};
options.sort = {'createdAt' : -1};
return this.findOne(query, options);
}
findPendingByUser(userId, options) {
const query = {
userId,
status: {
$nin: ['completed']
}
};
return this.find(query, options);
}
findAllPending(options) {
const query = {
status: { $nin: ['completed'] }
};
return this.find(query, options);
}
// UPDATE
updateOperation(data) {
const update = {
$set: {
roomList: data.roomList,
status: data.status,
fileList: data.fileList,
generatedFile: data.generatedFile
}
};
return this.update(data._id, update);
}
// INSERT
create(data) {
const exportOperation = {
createdAt: new Date
};
_.extend(exportOperation, data);
return this.insert(exportOperation);
}
// REMOVE
removeById(_id) {
return this.remove(_id);
}
};

@ -284,6 +284,14 @@ RocketChat.models.Messages = new class extends RocketChat.models._Base {
return this.findOne(query);
}
findByRoomId(roomId, options) {
const query = {
rid: roomId
};
return this.find(query, options);
}
getLastVisibleMessageSentWithNoTypeByRoomId(rid, messageId) {
const query = {
rid,

@ -0,0 +1,40 @@
import _ from 'underscore';
RocketChat.models.UserDataFiles = new class ModelUserDataFiles extends RocketChat.models._Base {
constructor() {
super('user_data_files');
this.tryEnsureIndex({ 'userId': 1 });
}
// FIND
findById(id) {
const query = {_id: id};
return this.find(query);
}
findLastFileByUser(userId, options = {}) {
const query = {
userId
};
options.sort = {'_updatedAt' : -1};
return this.findOne(query, options);
}
// INSERT
create(data) {
const userDataFile = {
createdAt: new Date
};
_.extend(userDataFile, data);
return this.insert(userDataFile);
}
// REMOVE
removeById(_id) {
return this.remove(_id);
}
};

@ -51,6 +51,11 @@ class ModelUsers extends RocketChat.models._Base {
return this.findOne(query, options);
}
findOneById(userId) {
const query = {_id: userId};
return this.findOne(query);
}
// FIND
findById(userId) {

@ -310,6 +310,19 @@
</div>
</div>
</div>
{{#if userDataDownloadEnabled}}
<div class="section">
<h1>{{_ "My Data"}}</h1>
<div class="section-content border-component-color">
<div class="input-line">
<label><button class="button download-my-data"><i class="icon-download secondary-font-color"></i> <span>{{_ "Download_My_Data"}}</span></button></label>
<label><button class="button export-my-data"><i class="icon-download secondary-font-color"></i> <span>{{_ "Export_My_Data"}}</span></button></label>
</div>
</div>
</div>
{{/if}}
</fieldset>
</form>
</div>

@ -87,6 +87,9 @@ Template.accountPreferences.helpers({
showRoles() {
return RocketChat.settings.get('UI_DisplayRoles');
},
userDataDownloadEnabled() {
return RocketChat.settings.get('UserData_EnableDownload') !== false;
},
notificationsSoundVolume() {
return RocketChat.getUserPreference(Meteor.user(), 'notificationsSoundVolume');
}
@ -202,6 +205,55 @@ Template.accountPreferences.onCreated(function() {
}
});
};
this.downloadMyData = function(fullExport = false) {
Meteor.call('requestDataDownload', {fullExport}, function(error, results) {
if (results) {
if (results.requested) {
modal.open({
title: t('UserDataDownload_Requested'),
text: t('UserDataDownload_Requested_Text'),
type: 'success'
});
return true;
}
if (results.exportOperation) {
if (results.exportOperation.status === 'completed') {
modal.open({
title: t('UserDataDownload_Requested'),
text: t('UserDataDownload_CompletedRequestExisted_Text'),
type: 'success'
});
return true;
}
modal.open({
title: t('UserDataDownload_Requested'),
text: t('UserDataDownload_RequestExisted_Text'),
type: 'success'
});
return true;
}
modal.open({
title: t('UserDataDownload_Requested'),
type: 'success'
});
return true;
}
if (error) {
return handleError(error);
}
});
};
this.exportMyData = function() {
this.downloadMyData(true);
};
});
Template.accountPreferences.onRendered(function() {
@ -221,6 +273,14 @@ Template.accountPreferences.events({
'click .enable-notifications'() {
KonchatNotification.getDesktopPermission();
},
'click .download-my-data'(e, t) {
e.preventDefault();
t.downloadMyData();
},
'click .export-my-data'(e, t) {
e.preventDefault();
t.exportMyData();
},
'click .test-notifications'(e) {
e.preventDefault();
KonchatNotification.notify({

@ -0,0 +1,18 @@
Package.describe({
name: 'rocketchat:user-data-download',
version: '1.0.0',
summary: 'Adds setting to allow the user to download all their data stored in the servers.',
git: ''
});
Package.onUse(function(api) {
api.use([
'ecmascript',
'rocketchat:file',
'rocketchat:lib',
'webapp'
]);
api.addFiles('server/startup/settings.js', 'server');
api.addFiles('server/cronProcessDownloads.js', 'server');
});

@ -0,0 +1,476 @@
/* globals SyncedCron */
import fs from 'fs';
import path from 'path';
import archiver from 'archiver';
let zipFolder = '/tmp/zipFiles';
if (RocketChat.settings.get('UserData_FileSystemZipPath') != null) {
if (RocketChat.settings.get('UserData_FileSystemZipPath').trim() !== '') {
zipFolder = RocketChat.settings.get('UserData_FileSystemZipPath');
}
}
let processingFrequency = 15;
if (RocketChat.settings.get('UserData_ProcessingFrequency') > 0) {
processingFrequency = RocketChat.settings.get('UserData_ProcessingFrequency');
}
const startFile = function(fileName, content) {
fs.writeFileSync(fileName, content);
};
const writeToFile = function(fileName, content) {
fs.appendFileSync(fileName, content);
};
const createDir = function(folderName) {
if (!fs.existsSync(folderName)) {
fs.mkdirSync(folderName);
}
};
const loadUserSubscriptions = function(exportOperation) {
exportOperation.roomList = [];
const exportUserId = exportOperation.userId;
const cursor = RocketChat.models.Subscriptions.findByUserId(exportUserId);
cursor.forEach((subscription) => {
const roomId = subscription.rid;
const roomData = subscription._room;
let roomName = roomData.name ? roomData.name : roomId;
let userId = null;
if (subscription.t === 'd') {
userId = roomId.replace(exportUserId, '');
const userData = RocketChat.models.Users.findOneById(userId);
if (userData) {
roomName = userData.name;
}
}
const fileName = exportOperation.fullExport ? roomId : roomName;
const fileType = exportOperation.fullExport ? 'json' : 'html';
const targetFile = `${ fileName }.${ fileType }`;
exportOperation.roomList.push({
roomId,
roomName,
userId,
exportedCount: 0,
status: 'pending',
targetFile,
type: subscription.t
});
});
if (exportOperation.fullExport) {
exportOperation.status = 'exporting-rooms';
} else {
exportOperation.status = 'exporting';
}
};
const getAttachmentData = function(attachment) {
const attachmentData = {
type : attachment.type,
title: attachment.title,
title_link: attachment.title_link,
image_url: attachment.image_url,
audio_url: attachment.audio_url,
video_url: attachment.video_url,
message_link: attachment.message_link,
image_type: attachment.image_type,
image_size: attachment.image_size,
video_size: attachment.video_size,
video_type: attachment.video_type,
audio_size: attachment.audio_size,
audio_type: attachment.audio_type,
url: null,
remote: false,
fileId: null,
fileName: null
};
const url = attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url || attachment.message_link;
if (url) {
attachmentData.url = url;
const urlMatch = /\:\/\//.exec(url);
if (urlMatch && urlMatch.length > 0) {
attachmentData.remote = true;
} else {
const match = /^\/([^\/]+)\/([^\/]+)\/(.*)/.exec(url);
if (match && match[2]) {
const file = RocketChat.models.Uploads.findOneById(match[2]);
if (file) {
attachmentData.fileId = file._id;
attachmentData.fileName = file.name;
}
}
}
}
return attachmentData;
};
const addToFileList = function(exportOperation, attachment) {
const targetFile = path.join(exportOperation.assetsPath, `${ attachment.fileId }-${ attachment.fileName }`);
const attachmentData = {
url: attachment.url,
copied: false,
remote: attachment.remote,
fileId: attachment.fileId,
fileName: attachment.fileName,
targetFile
};
exportOperation.fileList.push(attachmentData);
};
const getMessageData = function(msg, exportOperation) {
const attachments = [];
if (msg.attachments) {
msg.attachments.forEach((attachment) => {
const attachmentData = getAttachmentData(attachment);
attachments.push(attachmentData);
addToFileList(exportOperation, attachmentData);
});
}
const messageObject = {
msg: msg.msg,
username: msg.u.username,
ts: msg.ts
};
if (attachments && attachments.length > 0) {
messageObject.attachments = attachments;
}
if (msg.t) {
messageObject.type = msg.t;
}
if (msg.u.name) {
messageObject.name = msg.u.name;
}
return messageObject;
};
const copyFile = function(exportOperation, attachmentData) {
if (attachmentData.copied || attachmentData.remote || !attachmentData.fileId) {
attachmentData.copied = true;
return;
}
const file = RocketChat.models.Uploads.findOneById(attachmentData.fileId);
if (file) {
if (FileUpload.copy(file, attachmentData.targetFile)) {
attachmentData.copied = true;
}
}
};
const continueExportingRoom = function(exportOperation, exportOpRoomData) {
createDir(exportOperation.exportPath);
createDir(exportOperation.assetsPath);
const filePath = path.join(exportOperation.exportPath, exportOpRoomData.targetFile);
if (exportOpRoomData.status === 'pending') {
exportOpRoomData.status = 'exporting';
startFile(filePath, '');
if (!exportOperation.fullExport) {
writeToFile(filePath, '<meta http-equiv="content-type" content="text/html; charset=utf-8">');
}
}
let limit = 100;
if (RocketChat.settings.get('UserData_MessageLimitPerRequest') > 0) {
limit = RocketChat.settings.get('UserData_MessageLimitPerRequest');
}
const skip = exportOpRoomData.exportedCount;
const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip });
const count = cursor.count();
cursor.forEach((msg) => {
const messageObject = getMessageData(msg, exportOperation);
if (exportOperation.fullExport) {
const messageString = JSON.stringify(messageObject);
writeToFile(filePath, `${ messageString }\n`);
} else {
const messageType = msg.t;
const userName = msg.u.username || msg.u.name;
const timestamp = msg.ts ? new Date(msg.ts).toUTCString() : '';
let message = msg.msg;
switch (messageType) {
case 'uj':
message = TAPi18n.__('User_joined_channel');
break;
case 'ul':
message = TAPi18n.__('User_left');
break;
case 'au':
message = TAPi18n.__('User_added_by', {user_added : msg.msg, user_by : msg.u.username });
break;
case 'r':
message = TAPi18n.__('Room_name_changed', { room_name: msg.msg, user_by: msg.u.username });
break;
case 'ru':
message = TAPi18n.__('User_removed_by', {user_removed : msg.msg, user_by : msg.u.username });
break;
case 'wm':
message = TAPi18n.__('Welcome', {user: msg.u.username });
break;
case 'livechat-close':
message = TAPi18n.__('Conversation_finished');
break;
}
if (message !== msg.msg) {
message = `<i>${ message }</i>`;
}
writeToFile(filePath, `<p><strong>${ userName }</strong> (${ timestamp }):<br/>`);
writeToFile(filePath, message);
if (messageObject.attachments && messageObject.attachments.length > 0) {
messageObject.attachments.forEach((attachment) => {
if (attachment.type === 'file') {
const description = attachment.description || attachment.title || TAPi18n.__('Message_Attachments');
const assetUrl = `./assets/${ attachment.fileId }-${ attachment.fileName }`;
const link = `<br/><a href="${ assetUrl }">${ description }</a>`;
writeToFile(filePath, link);
}
});
}
writeToFile(filePath, '</p>');
}
exportOpRoomData.exportedCount++;
});
if (count <= exportOpRoomData.exportedCount) {
exportOpRoomData.status = 'completed';
return true;
}
return false;
};
const isExportComplete = function(exportOperation) {
const incomplete = exportOperation.roomList.some((exportOpRoomData) => {
return exportOpRoomData.status !== 'completed';
});
return !incomplete;
};
const isDownloadFinished = function(exportOperation) {
const anyDownloadPending = exportOperation.fileList.some((fileData) => {
return !fileData.copied && !fileData.remote;
});
return !anyDownloadPending;
};
const sendEmail = function(userId) {
const lastFile = RocketChat.models.UserDataFiles.findLastFileByUser(userId);
if (lastFile) {
const userData = RocketChat.models.Users.findOneById(userId);
if (userData && userData.emails && userData.emails[0] && userData.emails[0].address) {
const emailAddress = `${ userData.name } <${ userData.emails[0].address }>`;
const fromAddress = RocketChat.settings.get('From_Email');
const subject = TAPi18n.__('UserDataDownload_EmailSubject');
const download_link = lastFile.url;
const body = TAPi18n.__('UserDataDownload_EmailBody', { download_link });
const rfcMailPatternWithName = /^(?:.*<)?([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)(?:>?)$/;
if (rfcMailPatternWithName.test(emailAddress)) {
Meteor.defer(function() {
return Email.send({
to: emailAddress,
from: fromAddress,
subject,
html: body
});
});
return console.log(`Sending email to ${ emailAddress }`);
}
}
}
};
const makeZipFile = function(exportOperation) {
const targetFile = path.join(zipFolder, `${ exportOperation.userId }.zip`);
const output = fs.createWriteStream(targetFile);
exportOperation.generatedFile = targetFile;
const archive = archiver('zip');
output.on('close', () => {
});
archive.on('error', (err) => {
throw err;
});
archive.pipe(output);
archive.directory(exportOperation.exportPath, false);
archive.finalize();
};
const uploadZipFile = function(exportOperation, callback) {
const userDataStore = FileUpload.getStore('UserDataFiles');
const filePath = exportOperation.generatedFile;
const stat = Meteor.wrapAsync(fs.stat)(filePath);
const stream = fs.createReadStream(filePath);
const contentType = 'application/zip';
const size = stat.size;
const userId = exportOperation.userId;
const user = RocketChat.models.Users.findOneById(userId);
const userDisplayName = user ? user.name : userId;
const utcDate = new Date().toISOString().split('T')[0];
const newFileName = encodeURIComponent(`${ utcDate }-${ userDisplayName }.zip`);
const details = {
userId,
type: contentType,
size,
name: newFileName
};
userDataStore.insert(details, stream, (err) => {
if (err) {
throw new Meteor.Error('invalid-file', 'Invalid Zip File', { method: 'cronProcessDownloads.uploadZipFile' });
} else {
callback();
}
});
};
const generateChannelsFile = function(exportOperation) {
if (exportOperation.fullExport) {
const fileName = path.join(exportOperation.exportPath, 'channels.json');
startFile(fileName, '');
exportOperation.roomList.forEach((roomData) => {
const newRoomData = {
roomId: roomData.roomId,
roomName: roomData.roomName,
type: roomData.type
};
const messageString = JSON.stringify(newRoomData);
writeToFile(fileName, `${ messageString }\n`);
});
}
exportOperation.status = 'exporting';
};
const continueExportOperation = function(exportOperation) {
if (exportOperation.status === 'completed') {
return;
}
if (!exportOperation.roomList) {
loadUserSubscriptions(exportOperation);
}
try {
if (exportOperation.status === 'exporting-rooms') {
generateChannelsFile(exportOperation);
}
//Run every room on every request, to avoid missing new messages on the rooms that finished first.
if (exportOperation.status === 'exporting') {
exportOperation.roomList.forEach((exportOpRoomData) => {
continueExportingRoom(exportOperation, exportOpRoomData);
});
if (isExportComplete(exportOperation)) {
exportOperation.status = 'downloading';
return;
}
}
if (exportOperation.status === 'downloading') {
exportOperation.fileList.forEach((attachmentData) => {
copyFile(exportOperation, attachmentData);
});
if (isDownloadFinished(exportOperation)) {
exportOperation.status = 'compressing';
return;
}
}
if (exportOperation.status === 'compressing') {
makeZipFile(exportOperation);
exportOperation.status = 'uploading';
return;
}
if (exportOperation.status === 'uploading') {
uploadZipFile(exportOperation, () => {
exportOperation.status = 'completed';
RocketChat.models.ExportOperations.updateOperation(exportOperation);
});
return;
}
} catch (e) {
console.error(e);
}
};
function processDataDownloads() {
const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1});
cursor.forEach((exportOperation) => {
if (exportOperation.status === 'completed') {
return;
}
continueExportOperation(exportOperation);
RocketChat.models.ExportOperations.updateOperation(exportOperation);
if (exportOperation.status === 'completed') {
sendEmail(exportOperation.userId);
}
});
}
Meteor.startup(function() {
Meteor.defer(function() {
processDataDownloads();
SyncedCron.add({
name: 'Generate download files for user data',
schedule: (parser) => parser.cron(`*/${ processingFrequency } * * * *`),
job: processDataDownloads
});
});
});

@ -0,0 +1,34 @@
RocketChat.settings.addGroup('UserDataDownload', function() {
this.add('UserData_EnableDownload', true, {
type: 'boolean',
public: true,
i18nLabel: 'UserData_EnableDownload'
});
this.add('UserData_FileSystemPath', '', {
type: 'string',
public: true,
i18nLabel: 'UserData_FileSystemPath'
});
this.add('UserData_FileSystemZipPath', '', {
type: 'string',
public: true,
i18nLabel: 'UserData_FileSystemZipPath'
});
this.add('UserData_ProcessingFrequency', 15, {
type: 'int',
public: true,
i18nLabel: 'UserData_ProcessingFrequency'
});
this.add('UserData_MessageLimitPerRequest', 100, {
type: 'int',
public: true,
i18nLabel: 'UserData_MessageLimitPerRequest'
});
});

@ -0,0 +1,63 @@
import fs from 'fs';
import path from 'path';
let tempFolder = '/tmp/userData';
if (RocketChat.settings.get('UserData_FileSystemPath') != null) {
if (RocketChat.settings.get('UserData_FileSystemPath').trim() !== '') {
tempFolder = RocketChat.settings.get('UserData_FileSystemPath');
}
}
Meteor.methods({
requestDataDownload({fullExport = false}) {
const currentUserData = Meteor.user();
const userId = currentUserData._id;
const lastOperation = RocketChat.models.ExportOperations.findLastOperationByUser(userId, fullExport);
if (lastOperation) {
const yesterday = new Date();
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
if (lastOperation.createdAt > yesterday) {
return {
requested: false,
exportOperation: lastOperation
};
}
}
const subFolderName = fullExport ? 'full' : 'partial';
const baseFolder = path.join(tempFolder, userId);
if (!fs.existsSync(baseFolder)) {
fs.mkdirSync(baseFolder);
}
const folderName = path.join(baseFolder, subFolderName);
if (!fs.existsSync(folderName)) {
fs.mkdirSync(folderName);
}
const assetsFolder = path.join(folderName, 'assets');
if (!fs.existsSync(assetsFolder)) {
fs.mkdirSync(assetsFolder);
}
const exportOperation = {
userId : currentUserData._id,
roomList: null,
status: 'pending',
exportPath: folderName,
assetsPath: assetsFolder,
fileList: [],
generatedFile: null,
fullExport
};
RocketChat.models.ExportOperations.create(exportOperation);
return {
requested: true,
exportOperation
};
}
});

@ -18,12 +18,10 @@ RocketChat.Migrations.add({
}
if (RocketChat.models.Users) {
RocketChat.models.Users.find({ 'settings.preferences.viewMode': { $exists: 1 } }).forEach(function(user) {
RocketChat.models.Users.update(
{ _id: user._id },
{ $rename: { 'settings.preferences.viewMode': 'user.settings.preferences.messageViewMode' } },
);
});
RocketChat.models.Users.update(
{ 'settings.preferences.viewMode': { $exists: 1 } },
{ $rename: { 'settings.preferences.viewMode': 'user.settings.preferences.messageViewMode' } },
);
}
}
},
@ -45,12 +43,10 @@ RocketChat.Migrations.add({
}
if (RocketChat.models.Users) {
RocketChat.models.Users.find({ 'settings.preferences.messageViewMode': { $exists: 1 } }).forEach(function(user) {
RocketChat.models.Users.update(
{ _id: user._id },
{ $rename: { 'settings.preferences.messageViewMode': 'user.settings.preferences.viewMode' } },
);
});
RocketChat.models.Users.update(
{ 'settings.preferences.messageViewMode': { $exists: 1 } },
{ $rename: { 'settings.preferences.messageViewMode': 'user.settings.preferences.viewMode' } },
);
}
}
}

Loading…
Cancel
Save