Move Attachment to other storages now possible

client/components/cards/attachments.jade

@@ -72,3 +72,16 @@ template(name="attachmentActionsPopup")
         a.js-confirm-delete
           i.fa.fa-close
           | {{_ 'delete'}}
+        p.attachment-storage
+          | {{versions.original.storage}}
+
+        if $neq versions.original.storage "fs"
+          a.js-move-storage-fs
+            i.fa.fa-arrow-right
+            | {{_ 'attachment-move-storage-fs'}}
+
+        if $neq versions.original.storage "gridfs"
+          if versions.original.storage
+            a.js-move-storage-gridfs
+              i.fa.fa-arrow-right
+              | {{_ 'attachment-move-storage-gridfs'}}

client/components/cards/attachments.js

@@ -138,6 +138,14 @@ BlazeComponent.extendComponent({
           Cards.findOne(this.data().meta.cardId).unsetCover();
           Popup.back();
         },
+        'click .js-move-storage-fs'() {
+          Meteor.call('moveToStorage', this.data()._id, "fs");
+          Popup.back();
+        },
+        'click .js-move-storage-gridfs'() {
+          Meteor.call('moveToStorage', this.data()._id, "gridfs");
+          Popup.back();
+        },
       }
     ]
   }

i18n/en.i18n.json

@@ -1165,5 +1165,7 @@
   "copyChecklistPopup-title": "Copy Checklist",
   "card-show-lists": "Card Show Lists",
   "subtaskActionsPopup-title": "Subtask Actions",
-  "attachmentActionsPopup-title": "Attachment Actions"
+  "attachmentActionsPopup-title": "Attachment Actions",
+  "attachment-move-storage-fs": "Move attachment to filesystem",
+  "attachment-move-storage-gridfs": "Move attachment to GridFS"
 }

models/attachments.js

@@ -2,30 +2,7 @@ import { Meteor } from 'meteor/meteor';
 import { FilesCollection } from 'meteor/ostrio:files';
 import fs from 'fs';
 import path from 'path';
-import { createBucket } from './lib/grid/createBucket';
-import { createOnAfterUpload } from './lib/fsHooks/createOnAfterUpload';
-import { createInterceptDownload } from './lib/fsHooks/createInterceptDownload';
-import { createOnAfterRemove } from './lib/fsHooks/createOnAfterRemove';
-
-let attachmentBucket;
-if (Meteor.isServer) {
-  attachmentBucket = createBucket('attachments');
-}
-
-const insertActivity = (fileObj, activityType) =>
-  Activities.insert({
-    userId: fileObj.userId,
-    type: 'card',
-    activityType,
-    attachmentId: fileObj._id,
-    // this preserves the name so that notifications can be meaningful after
-    // this file is removed
-    attachmentName: fileObj.name,
-    boardId: fileObj.meta.boardId,
-    cardId: fileObj.meta.cardId,
-    listId: fileObj.meta.listId,
-    swimlaneId: fileObj.meta.swimlaneId,
-  });
+import AttachmentStoreStrategy from '/models/lib/attachmentStoreStrategy';
 
 // XXX Enforce a schema for the Attachments FilesCollection
 // see: https://github.com/VeliovGroup/Meteor-Files/wiki/Schema
@@ -40,20 +17,20 @@ Attachments = new FilesCollection({
     }
     return path.normalize(`assets/app/uploads/${this.collectionName}`);
   },
-  onAfterUpload: function onAfterUpload(fileRef) {
-    createOnAfterUpload(attachmentBucket).call(this, fileRef);
-    // If the attachment doesn't have a source field
-    // or its source is different than import
-    if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
-      // Add activity about adding the attachment
-      insertActivity(fileRef, 'addAttachment');
-    }
+  onAfterUpload(fileObj) {
+    Object.keys(fileObj.versions).forEach(versionName => {
+      AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).onAfterUpload();
+    })
+  },
+  interceptDownload(http, fileObj, versionName) {
+    const ret = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).interceptDownload(http);
+    return ret;
   },
-  interceptDownload: createInterceptDownload(attachmentBucket),
-  onAfterRemove: function onAfterRemove(files) {
-    createOnAfterRemove(attachmentBucket).call(this, files);
+  onAfterRemove(files) {
     files.forEach(fileObj => {
-      insertActivity(fileObj, 'deleteAttachment');
+      Object.keys(fileObj.versions).forEach(versionName => {
+        AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName).onAfterRemove();
+      });
     });
   },
   // We authorize the attachment download either:
@@ -82,6 +59,45 @@ if (Meteor.isServer) {
     fetch: ['meta'],
   });
 
+  Meteor.methods({
+    moveToStorage(fileObjId, storageDestination) {
+      check(fileObjId, String);
+      check(storageDestination, String);
+
+      const fileObj = Attachments.findOne({_id: fileObjId});
+
+      Object.keys(fileObj.versions).forEach(versionName => {
+        const strategyRead = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName);
+        const strategyWrite = AttachmentStoreStrategy.getFileStrategy(this, fileObj, versionName, storageDestination);
+
+        if (strategyRead.constructor.name != strategyWrite.constructor.name) {
+          const readStream = strategyRead.getReadStream();
+          const writeStream = strategyWrite.getWriteStream();
+
+          writeStream.on('error', error => {
+            console.error('[writeStream error]: ', error, fileObjId);
+          });
+
+          readStream.on('error', error => {
+            console.error('[readStream error]: ', error, fileObjId);
+          });
+
+          writeStream.on('finish', Meteor.bindEnvironment((finishedData) => {
+            strategyWrite.writeStreamFinished(finishedData);
+          }));
+
+          // https://forums.meteor.com/t/meteor-code-must-always-run-within-a-fiber-try-wrapping-callbacks-that-you-pass-to-non-meteor-libraries-with-meteor-bindenvironmen/40099/8
+          readStream.on('end', Meteor.bindEnvironment(() => {
+            Attachments.update({ _id: fileObj._id }, { $set: { [`versions.${versionName}.storage`]: strategyWrite.getStorageName() } });
+            strategyRead.unlink();
+          }));
+
+          readStream.pipe(writeStream);
+        }
+      });
+    },
+  });
+
   Meteor.startup(() => {
     Attachments.collection._ensureIndex({ 'meta.cardId': 1 });
     const storagePath = Attachments.storagePath();

models/avatars.js

@@ -28,9 +28,22 @@ Avatars = new FilesCollection({
     }
     return 'avatar-too-big';
   },
-  onAfterUpload: createOnAfterUpload(avatarsBucket),
-  interceptDownload: createInterceptDownload(avatarsBucket),
-  onAfterRemove: createOnAfterRemove(avatarsBucket),
+  onAfterUpload(fileObj) {
+    Object.keys(fileObj.versions).forEach(versionName => {
+      createOnAfterUpload(this, avatarsBucket, fileObj, versionName);
+    });
+  },
+  interceptDownload(http, fileObj, versionName) {
+    const ret = createInterceptDownload(this, avatarsBucket, fileObj, http, versionName);
+    return ret;
+  },
+  onAfterRemove(files) {
+    files.forEach(fileObj => {
+      Object.keys(fileObj.versions).forEach(versionName => {
+        createOnAfterRemove(this, avatarsBucket, fileObj, versionName);
+      });
+    });
+  },
 });
 
 function isOwner(userId, doc) {

models/lib/attachmentStoreStrategy.js

@@ -0,0 +1,246 @@
+import fs from 'fs';
+import { createBucket } from './grid/createBucket';
+import { createObjectId } from './grid/createObjectId';
+import { createOnAfterUpload } from './fsHooks/createOnAfterUpload';
+import { createInterceptDownload } from './fsHooks/createInterceptDownload';
+import { createOnAfterRemove } from './fsHooks/createOnAfterRemove';
+
+const insertActivity = (fileObj, activityType) =>
+  Activities.insert({
+    userId: fileObj.userId,
+    type: 'card',
+    activityType,
+    attachmentId: fileObj._id,
+    attachmentName: fileObj.name,
+    boardId: fileObj.meta.boardId,
+    cardId: fileObj.meta.cardId,
+    listId: fileObj.meta.listId,
+    swimlaneId: fileObj.meta.swimlaneId,
+  });
+
+let attachmentBucket;
+if (Meteor.isServer) {
+  attachmentBucket = createBucket('attachments');
+}
+
+/** Strategy to store attachments */
+class AttachmentStoreStrategy {
+
+  /** constructor
+   * @param filesCollection the current FilesCollection instance
+   * @param fileObj the current file object
+   * @param versionName the current version
+   */
+  constructor(filesCollection, fileObj, versionName) {
+    this.filesCollection = filesCollection;
+    this.fileObj = fileObj;
+    this.versionName = versionName;
+  }
+
+  /** after successfull upload */
+  onAfterUpload() {
+    // If the attachment doesn't have a source field or its source is different than import
+    if (!this.fileObj.meta.source || this.fileObj.meta.source !== 'import') {
+      // Add activity about adding the attachment
+      insertActivity(this.fileObj, 'addAttachment');
+    }
+  }
+
+  /** download the file
+   * @param http the current http request
+   */
+  interceptDownload(http) {
+  }
+
+  /** after file remove */
+  onAfterRemove() {
+    insertActivity(this.fileObj, 'deleteAttachment');
+  }
+
+  /** returns a read stream
+   * @return the read stream
+   */
+  getReadStream() {
+  }
+
+  /** returns a write stream
+   * @return the write stream
+   */
+  getWriteStream() {
+  }
+
+  /** writing finished
+   * @param finishedData the data of the write stream finish event
+   */
+  writeStreamFinished(finishedData) {
+  }
+
+  /** remove the file */
+  unlink() {
+  }
+
+  /** return the storage name
+   * @return the storage name
+   */
+  getStorageName() {
+  }
+
+  static getFileStrategy(filesCollection, fileObj, versionName, storage) {
+    if (!storage) {
+      storage = fileObj.versions[versionName].storage || "gridfs";
+    }
+    let ret;
+    if (["fs", "gridfs"].includes(storage)) {
+      if (storage == "fs") {
+        ret = new AttachmentStoreStrategyFilesystem(filesCollection, fileObj, versionName);
+      } else if (storage == "gridfs") {
+        ret = new AttachmentStoreStrategyGridFs(filesCollection, fileObj, versionName);
+      }
+    }
+    console.log("getFileStrategy: ", ret.constructor.name);
+    return ret;
+  }
+}
+
+/** Strategy to store attachments at GridFS (MongoDB) */
+class AttachmentStoreStrategyGridFs extends AttachmentStoreStrategy {
+
+  /** constructor
+   * @param filesCollection the current FilesCollection instance
+   * @param fileObj the current file object
+   * @param versionName the current version
+   */
+  constructor(filesCollection, fileObj, versionName) {
+    super(filesCollection, fileObj, versionName);
+  }
+
+  /** after successfull upload */
+  onAfterUpload() {
+    createOnAfterUpload(this.filesCollection, attachmentBucket, this.fileObj, this.versionName);
+    super.onAfterUpload();
+  }
+
+  /** download the file
+   * @param http the current http request
+   */
+  interceptDownload(http) {
+    const ret = createInterceptDownload(this.filesCollection, attachmentBucket, this.fileObj, http, this.versionName);
+    return ret;
+  }
+
+  /** after file remove */
+  onAfterRemove() {
+    this.unlink();
+    super.onAfterRemove();
+  }
+
+  /** returns a read stream
+   * @return the read stream
+   */
+  getReadStream() {
+    const gridFsFileId = (this.fileObj.versions[this.versionName].meta || {})
+      .gridFsFileId;
+    let ret;
+    if (gridFsFileId) {
+      const gfsId = createObjectId({ gridFsFileId });
+      ret = attachmentBucket.openDownloadStream(gfsId);
+    }
+    return ret;
+  }
+
+  /** returns a write stream
+   * @return the write stream
+   */
+  getWriteStream() {
+    const fileObj = this.fileObj;
+    const versionName = this.versionName;
+    const metadata = { ...fileObj.meta, versionName, fileId: fileObj._id };
+    const ret = attachmentBucket.openUploadStream(this.fileObj.name, {
+      contentType: fileObj.type || 'binary/octet-stream',
+      metadata,
+    });
+    return ret;
+  }
+
+  /** writing finished
+   * @param finishedData the data of the write stream finish event
+   */
+  writeStreamFinished(finishedData) {
+    const gridFsFileIdName = this.getGridFsFileIdName();
+    Attachments.update({ _id: this.fileObj._id }, { $set: { [gridFsFileIdName]: finishedData._id.toHexString(), } });
+  }
+
+  /** remove the file */
+  unlink() {
+    createOnAfterRemove(this.filesCollection, attachmentBucket, this.fileObj, this.versionName);
+    const gridFsFileIdName = this.getGridFsFileIdName();
+    Attachments.update({ _id: this.fileObj._id }, { $unset: { [gridFsFileIdName]: 1 } });
+  }
+
+  /** return the storage name
+   * @return the storage name
+   */
+  getStorageName() {
+    return "gridfs";
+  }
+
+  /** returns the property name of gridFsFileId
+   * @return the property name of gridFsFileId
+   */
+  getGridFsFileIdName() {
+    const ret = `versions.${this.versionName}.meta.gridFsFileId`;
+    return ret;
+  }
+}
+
+/** Strategy to store attachments at filesystem */
+class AttachmentStoreStrategyFilesystem extends AttachmentStoreStrategy {
+
+  /** constructor
+   * @param filesCollection the current FilesCollection instance
+   * @param fileObj the current file object
+   * @param versionName the current version
+   */
+  constructor(filesCollection, fileObj, versionName) {
+    super(filesCollection, fileObj, versionName);
+  }
+
+  /** returns a read stream
+   * @return the read stream
+   */
+  getReadStream() {
+    const ret = fs.createReadStream(this.fileObj.versions[this.versionName].path)
+    return ret;
+  }
+
+  /** returns a write stream
+   * @return the write stream
+   */
+  getWriteStream() {
+    const newFileName = this.fileObj.name;
+    const filePath = this.fileObj.versions[this.versionName].path;
+    const ret = fs.createWriteStream(filePath);
+    return ret;
+  }
+
+  /** writing finished
+   * @param finishedData the data of the write stream finish event
+   */
+  writeStreamFinished(finishedData) {
+  }
+
+  /** remove the file */
+  unlink() {
+    const filePath = this.fileObj.versions[this.versionName].path;
+    fs.unlink(filePath, () => {});
+  }
+
+  /** return the storage name
+   * @return the storage name
+   */
+  getStorageName() {
+    return "fs";
+  }
+}
+
+export default AttachmentStoreStrategy;

models/lib/fsHooks/createInterceptDownload.js

@@ -1,7 +1,7 @@
 import { createObjectId } from '../grid/createObjectId';
 
-export const createInterceptDownload = bucket =>
-  function interceptDownload(http, file, versionName) {
+export const createInterceptDownload =
+  function interceptDownload(filesCollection, bucket, file, http, versionName) {
     const { gridFsFileId } = file.versions[versionName].meta || {};
     if (gridFsFileId) {
       // opens the download stream using a given gfs id
@@ -24,7 +24,7 @@ export const createInterceptDownload = bucket =>
         http.response.end('not found');
       });
 
-      http.response.setHeader('Cache-Control', this.cacheControl);
+      http.response.setHeader('Cache-Control', filesCollection.cacheControl);
       http.response.setHeader(
         'Content-Disposition',
         getContentDisposition(file.name, http?.params?.query?.download),

models/lib/fsHooks/createOnAfterRemove.js

@@ -1,17 +1,12 @@
 import { createObjectId } from '../grid/createObjectId';
 
-export const createOnAfterRemove = bucket =>
-  function onAfterRemove(files) {
-    files.forEach(file => {
-      Object.keys(file.versions).forEach(versionName => {
-        const gridFsFileId = (file.versions[versionName].meta || {})
-          .gridFsFileId;
-        if (gridFsFileId) {
-          const gfsId = createObjectId({ gridFsFileId });
-          bucket.delete(gfsId, err => {
-            // if (err) console.error(err);
-          });
-        }
+export const createOnAfterRemove =
+  function onAfterRemove(filesCollection, bucket, file, versionName) {
+    const gridFsFileId = (file.versions[versionName].meta || {})
+      .gridFsFileId;
+    if (gridFsFileId) {
+      const gfsId = createObjectId({ gridFsFileId });
+        bucket.delete(gfsId, err => {
       });
-    });
+    }
   };

models/lib/fsHooks/createOnAfterUpload.js

@@ -1,51 +1,42 @@
 import { Meteor } from 'meteor/meteor';
 import fs from 'fs';
 
-export const createOnAfterUpload = bucket =>
-  function onAfterUpload(file) {
-    const self = this;
-
-    // here you could manipulate your file
-    // and create a new version, for example a scaled 'thumbnail'
-    // ...
-
-    // then we read all versions we have got so far
-    Object.keys(file.versions).forEach(versionName => {
-      const metadata = { ...file.meta, versionName, fileId: file._id };
-      fs.createReadStream(file.versions[versionName].path)
-
-        // this is where we upload the binary to the bucket using bucket.openUploadStream
-        // see http://mongodb.github.io/node-mongodb-native/3.2/api/GridFSBucket.html#openUploadStream
-        .pipe(
-          bucket.openUploadStream(file.name, {
-            contentType: file.type || 'binary/octet-stream',
-            metadata,
-          }),
-        )
-
-        // and we unlink the file from the fs on any error
-        // that occurred during the upload to prevent zombie files
-        .on('error', err => {
-          console.error("[createOnAfterUpload error]", err);
-          self.unlink(this.collection.findOne(file._id), versionName); // Unlink files from FS
-        })
-
-        // once we are finished, we attach the gridFS Object id on the
-        // FilesCollection document's meta section and finally unlink the
-        // upload file from the filesystem
-        .on(
-          'finish',
-          Meteor.bindEnvironment(ver => {
-            const property = `versions.${versionName}.meta.gridFsFileId`;
-
-            self.collection.update(file._id, {
-              $set: {
-                [property]: ver._id.toHexString(),
-              },
-            });
-
-            self.unlink(this.collection.findOne(file._id), versionName); // Unlink files from FS
-          }),
-        );
-    });
-  };
+export const createOnAfterUpload = function onAfterUpload(filesCollection, bucket, file, versionName) {
+  const self = filesCollection;
+  const metadata = { ...file.meta, versionName, fileId: file._id };
+  fs.createReadStream(file.versions[versionName].path)
+
+    // this is where we upload the binary to the bucket using bucket.openUploadStream
+    // see http://mongodb.github.io/node-mongodb-native/3.2/api/GridFSBucket.html#openUploadStream
+    .pipe(
+      bucket.openUploadStream(file.name, {
+        contentType: file.type || 'binary/octet-stream',
+        metadata,
+      }),
+    )
+
+    // and we unlink the file from the fs on any error
+    // that occurred during the upload to prevent zombie files
+    .on('error', err => {
+      console.error("[createOnAfterUpload error]", err);
+      self.unlink(self.collection.findOne(file._id), versionName); // Unlink files from FS
+    })
+
+    // once we are finished, we attach the gridFS Object id on the
+    // FilesCollection document's meta section and finally unlink the
+    // upload file from the filesystem
+    .on(
+      'finish',
+      Meteor.bindEnvironment(ver => {
+        const property = `versions.${versionName}.meta.gridFsFileId`;
+
+        self.collection.update(file._id, {
+          $set: {
+            [property]: ver._id.toHexString(),
+          },
+        });
+
+        self.unlink(self.collection.findOne(file._id), versionName); // Unlink files from FS
+      }),
+    );
+};