From 438ac23e2a3564fbce50e862336db56bbf0d747c Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2020 22:09:19 +0200 Subject: [PATCH 1/4] Distribute preview folders in appdata in multibucket setup to multiple buckets * introduces a new IRootMountProvider to register mount points inside the root storage * adds a AppdataPreviewObjectStoreStorage to handle the split between preview folders and bucket number Ref #22033 Signed-off-by: Morris Jobke --- lib/composer/composer/autoload_classmap.php | 3 + lib/composer/composer/autoload_static.php | 3 + .../Files/Config/MountProviderCollection.php | 19 ++++ .../ObjectStorePreviewCacheMountProvider.php | 102 ++++++++++++++++++ .../AppdataPreviewObjectStoreStorage.php | 44 ++++++++ lib/private/Preview/Storage/Root.php | 1 - lib/private/Server.php | 3 + lib/private/legacy/OC_Util.php | 11 ++ .../Files/Config/IRootMountProvider.php | 41 +++++++ 9 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php create mode 100644 lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php create mode 100644 lib/public/Files/Config/IRootMountProvider.php diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 254840b3542..65542e477e7 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -230,6 +230,7 @@ return array( 'OCP\\Files\\Config\\IHomeMountProvider' => $baseDir . '/lib/public/Files/Config/IHomeMountProvider.php', 'OCP\\Files\\Config\\IMountProvider' => $baseDir . '/lib/public/Files/Config/IMountProvider.php', 'OCP\\Files\\Config\\IMountProviderCollection' => $baseDir . '/lib/public/Files/Config/IMountProviderCollection.php', + 'OCP\\Files\\Config\\IRootMountProvider' => $baseDir . '/lib/public/Files/Config/IRootMountProvider.php', 'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php', 'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php', 'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php', @@ -1029,6 +1030,7 @@ return array( 'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', + 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php', 'OC\\Files\\Node\\HookConnector' => $baseDir . '/lib/private/Files/Node/HookConnector.php', @@ -1040,6 +1042,7 @@ return array( 'OC\\Files\\Node\\Root' => $baseDir . '/lib/private/Files/Node/Root.php', 'OC\\Files\\Notify\\Change' => $baseDir . '/lib/private/Files/Notify/Change.php', 'OC\\Files\\Notify\\RenameChange' => $baseDir . '/lib/private/Files/Notify/RenameChange.php', + 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 45bf93241d5..835603a3340 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -259,6 +259,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OCP\\Files\\Config\\IHomeMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IHomeMountProvider.php', 'OCP\\Files\\Config\\IMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProvider.php', 'OCP\\Files\\Config\\IMountProviderCollection' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IMountProviderCollection.php', + 'OCP\\Files\\Config\\IRootMountProvider' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IRootMountProvider.php', 'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php', 'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php', 'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php', @@ -1058,6 +1059,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', + 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php', 'OC\\Files\\Node\\HookConnector' => __DIR__ . '/../../..' . '/lib/private/Files/Node/HookConnector.php', @@ -1069,6 +1071,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Files\\Node\\Root' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Root.php', 'OC\\Files\\Notify\\Change' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/Change.php', 'OC\\Files\\Notify\\RenameChange' => __DIR__ . '/../../..' . '/lib/private/Files/Notify/RenameChange.php', + 'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php', 'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php', 'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php', diff --git a/lib/private/Files/Config/MountProviderCollection.php b/lib/private/Files/Config/MountProviderCollection.php index 34db652290f..2b57ffe6e4c 100644 --- a/lib/private/Files/Config/MountProviderCollection.php +++ b/lib/private/Files/Config/MountProviderCollection.php @@ -30,6 +30,7 @@ use OC\Hooks\EmitterTrait; use OCP\Files\Config\IHomeMountProvider; use OCP\Files\Config\IMountProvider; use OCP\Files\Config\IMountProviderCollection; +use OCP\Files\Config\IRootMountProvider; use OCP\Files\Config\IUserMountCache; use OCP\Files\Mount\IMountManager; use OCP\Files\Mount\IMountPoint; @@ -49,6 +50,9 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { */ private $providers = []; + /** @var \OCP\Files\Config\IRootMountProvider[] */ + private $rootProviders = []; + /** * @var \OCP\Files\Storage\IStorageFactory */ @@ -198,4 +202,19 @@ class MountProviderCollection implements IMountProviderCollection, Emitter { public function getMountCache() { return $this->mountCache; } + + public function registerRootProvider(IRootMountProvider $provider) { + $this->rootProviders[] = $provider; + } + + public function getRootMounts(): array { + $loader = $this->loader; + $mounts = array_map(function (IRootMountProvider $provider) use ($loader) { + return $provider->getRootMounts($loader); + }, $this->rootProviders); + $mounts = array_reduce($mounts, function (array $mounts, array $providerMounts) { + return array_merge($mounts, $providerMounts); + }, []); + return $mounts; + } } diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php new file mode 100644 index 00000000000..a4acdb6bb0f --- /dev/null +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -0,0 +1,102 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\Mount; + +use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage; +use OCP\Files\Config\IRootMountProvider; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; +use OCP\ILogger; + +/** + * Mount provider for object store app data folder for previews + */ +class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { + /** @var ILogger */ + private $logger; + /** @var IConfig */ + private $config; + + public function __construct(ILogger $logger, IConfig $config) { + $this->logger = $logger; + $this->config = $config; + } + + public function getRootMounts(IStorageFactory $loader): array { + if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { + return []; + } + + $instanceId = $this->config->getSystemValueString('instanceid', ''); + $mountPoints = []; + $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; + $i = 0; + foreach ($directoryRange as $parent) { + foreach ($directoryRange as $child) { + $mountPoints[] = new MountPoint( + AppdataPreviewObjectStoreStorage::class, + '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, + $this->getMultiBucketObjectStore($i), + $loader + ); + $i++; + } + } + return $mountPoints; + } + + /** + * @return array + */ + protected function getMultiBucketObjectStore(int $number): array { + $config = $this->config->getSystemValue('objectstore_multibucket'); + + // sanity checks + if (empty($config['class'])) { + $this->logger->error('No class given for objectstore', ['app' => 'files']); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; + $config['arguments']['bucket'] .= (string)($number % $numBuckets); + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + $config['arguments']['internal-id'] = $number; + + return $config['arguments']; + } +} diff --git a/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php new file mode 100644 index 00000000000..ab6cb07cdc3 --- /dev/null +++ b/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php @@ -0,0 +1,44 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OC\Files\ObjectStore; + +class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { + + /** @var string */ + private $internalId; + + public function __construct($params) { + if (!isset($params['internal-id'])) { + throw new \Exception('missing id in parameters'); + } + $this->internalId = (string)$params['internal-id']; + parent::__construct($params); + } + + public function getId() { + return 'object::appdata::preview:' . $this->internalId; + } +} diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php index a9a72026a51..107d87c6301 100644 --- a/lib/private/Preview/Storage/Root.php +++ b/lib/private/Preview/Storage/Root.php @@ -1,7 +1,6 @@ * diff --git a/lib/private/Server.php b/lib/private/Server.php index e054d8230d9..8d771bec3a1 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -88,6 +88,7 @@ use OC\Files\Config\UserMountCacheListener; use OC\Files\Mount\CacheMountProvider; use OC\Files\Mount\LocalHomeMountProvider; use OC\Files\Mount\ObjectHomeMountProvider; +use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; use OC\Files\Node\HookConnector; use OC\Files\Node\LazyRoot; use OC\Files\Node\Root; @@ -904,9 +905,11 @@ class Server extends ServerContainer implements IServerContainer { // builtin providers $config = $c->getConfig(); + $logger = $c->getLogger(); $manager->registerProvider(new CacheMountProvider($config)); $manager->registerHomeProvider(new LocalHomeMountProvider()); $manager->registerHomeProvider(new ObjectHomeMountProvider($config)); + $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config)); return $manager; }); diff --git a/lib/private/legacy/OC_Util.php b/lib/private/legacy/OC_Util.php index ab386ab6172..fd55962447e 100644 --- a/lib/private/legacy/OC_Util.php +++ b/lib/private/legacy/OC_Util.php @@ -297,6 +297,17 @@ class OC_Util { self::initLocalStorageRootFS(); } + /** @var \OCP\Files\Config\IMountProviderCollection $mountProviderCollection */ + $mountProviderCollection = \OC::$server->query(\OCP\Files\Config\IMountProviderCollection::class); + /** @var \OCP\Files\Mount\IMountPoint[] $rootMountProviders */ + $rootMountProviders = $mountProviderCollection->getRootMounts(); + + /** @var \OC\Files\Mount\Manager $mountManager */ + $mountManager = \OC\Files\Filesystem::getMountManager(); + foreach ($rootMountProviders as $rootMountProvider) { + $mountManager->addMount($rootMountProvider); + } + if ($user != '' && !\OC::$server->getUserManager()->userExists($user)) { \OC::$server->getEventLogger()->end('setup_fs'); return false; diff --git a/lib/public/Files/Config/IRootMountProvider.php b/lib/public/Files/Config/IRootMountProvider.php new file mode 100644 index 00000000000..0f7b0eca3d4 --- /dev/null +++ b/lib/public/Files/Config/IRootMountProvider.php @@ -0,0 +1,41 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCP\Files\Config; + +use OCP\Files\Storage\IStorageFactory; + +/** + * @since 20.0.0 + */ +interface IRootMountProvider { + /** + * Get all root mountpoints + * + * @return \OCP\Files\Mount\IMountPoint[] + * @since 20.0.0 + */ + public function getRootMounts(IStorageFactory $loader): array; +} From 159f28cd521e94eff23e32b5bf3f4d2447da403f Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 30 Jul 2020 23:53:54 +0200 Subject: [PATCH 2/4] Mount the old previews in a separate folder for the multi bucket setup and check in them before using the actual locations Signed-off-by: Morris Jobke --- .../ObjectStorePreviewCacheMountProvider.php | 51 ++++++++- lib/private/Preview/Storage/Root.php | 11 ++ ...jectStorePreviewCacheMountProviderTest.php | 106 ++++++++++++++++++ 3 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php index a4acdb6bb0f..9ab0327684b 100644 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -26,6 +26,8 @@ declare(strict_types=1); namespace OC\Files\Mount; use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage; +use OC\Files\ObjectStore\ObjectStoreStorage; +use OC\Files\Storage\Wrapper\Jail; use OCP\Files\Config\IRootMountProvider; use OCP\Files\Storage\IStorageFactory; use OCP\IConfig; @@ -45,6 +47,10 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { $this->config = $config; } + /** + * @return MountPoint[] + * @throws \Exception + */ public function getRootMounts(IStorageFactory $loader): array { if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { return []; @@ -65,12 +71,25 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { $i++; } } + + $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); + $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); + $fakeRootStorageJail = new Jail([ + 'storage' => $fakeRootStorage, + 'root' => '/appdata_' . $instanceId . '/preview', + ]); + + // add a fallback location to be able to fetch existing previews from the old bucket + $mountPoints[] = new MountPoint( + $fakeRootStorageJail, + '/appdata_' . $instanceId . '/preview/old-multibucket', + null, + $loader + ); + return $mountPoints; } - /** - * @return array - */ protected function getMultiBucketObjectStore(int $number): array { $config = $this->config->getSystemValue('objectstore_multibucket'); @@ -99,4 +118,30 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { return $config['arguments']; } + + protected function getMultiBucketObjectStoreForRoot(): array { + $config = $this->config->getSystemValue('objectstore_multibucket'); + + // sanity checks + if (empty($config['class'])) { + $this->logger->error('No class given for objectstore', ['app' => 'files']); + } + if (!isset($config['arguments'])) { + $config['arguments'] = []; + } + + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + $config['arguments']['bucket'] .= '0'; + + // instantiate object store implementation + $config['arguments']['objectstore'] = new $config['class']($config['arguments']); + + return $config['arguments']; + } } diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php index 107d87c6301..37ae1758121 100644 --- a/lib/private/Preview/Storage/Root.php +++ b/lib/private/Preview/Storage/Root.php @@ -40,6 +40,17 @@ class Root extends AppData { public function getFolder(string $name): ISimpleFolder { $internalFolder = $this->getInternalFolder($name); + try { + return parent::getFolder('old-multibucket/' . $internalFolder); + } catch (NotFoundException $e) { + // not in multibucket fallback #1 + } + try { + return parent::getFolder('old-multibucket/' . $name); + } catch (NotFoundException $e) { + // not in multibucket fallback #2 + } + try { return parent::getFolder($internalFolder); } catch (NotFoundException $e) { diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php new file mode 100644 index 00000000000..2da07393f40 --- /dev/null +++ b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php @@ -0,0 +1,106 @@ + + * + * @author Morris Jobke + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace Test\Files\Mount; + +use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; +use OC\Files\ObjectStore\S3; +use OC\Files\Storage\StorageFactory; +use OCP\Files\Storage\IStorageFactory; +use OCP\IConfig; +use OCP\ILogger; +use PHPUnit\Framework\MockObject\MockObject; + +/** + * @group DB + * + * The DB permission is needed for the fake root storage initialization + */ +class ObjectStorePreviewCacheMountProviderTest extends \Test\TestCase { + + /** @var ObjectStorePreviewCacheMountProvider */ + protected $provider; + + /** @var ILogger|MockObject */ + protected $logger; + /** @var IConfig|MockObject */ + protected $config; + /** @var IStorageFactory|MockObject */ + protected $loader; + + + protected function setUp(): void { + parent::setUp(); + + $this->logger = $this->createMock(ILogger::class); + $this->config = $this->createMock(IConfig::class); + $this->loader = $this->createMock(StorageFactory::class); + + $this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config); + } + + public function testNoMultibucketObjectStorage() { + $this->config->expects($this->once()) + ->method('getSystemValue') + ->with('objectstore_multibucket') + ->willReturn(null); + + $this->assertEquals([], $this->provider->getRootMounts($this->loader)); + } + + public function testMultibucketObjectStorage() { + $this->config->expects($this->any()) + ->method('getSystemValue') + ->with('objectstore_multibucket') + ->willReturn([ + 'class' => S3::class, + 'arguments' => [ + 'bucket' => 'abc', + 'num_buckets' => 64, + 'key' => 'KEY', + 'secret' => 'SECRET', + 'hostname' => 'IP', + 'port' => 'PORT', + 'use_ssl' => false, + 'use_path_style' => true, + ], + ]); + $this->config->expects($this->once()) + ->method('getSystemValueString') + ->with('instanceid') + ->willReturn('INSTANCEID'); + + $mounts = $this->provider->getRootMounts($this->loader); + + // 256 mounts for the subfolders and 1 for the fake root + $this->assertCount(257, $mounts); + + // do some sanity checks if they have correct mount point paths + $this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint()); + $this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint()); + // also test the path of the fake bucket + $this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint()); + } +} From 4fdd38c7378415c552859dc4c0d2dda709f699ec Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 6 Aug 2020 14:00:51 +0200 Subject: [PATCH 3/4] Use fixed preview buckets that are postfixed with -preview-NUMBER Signed-off-by: Morris Jobke --- .../Files/Mount/ObjectStorePreviewCacheMountProvider.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php index 9ab0327684b..dba1dfc28e5 100644 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -108,8 +108,8 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { if (!isset($config['arguments']['bucket'])) { $config['arguments']['bucket'] = ''; } - $numBuckets = isset($config['arguments']['num_buckets']) ? $config['arguments']['num_buckets'] : 64; - $config['arguments']['bucket'] .= (string)($number % $numBuckets); + + $config['arguments']['bucket'] .= "-preview-$number"; // instantiate object store implementation $config['arguments']['objectstore'] = new $config['class']($config['arguments']); From 45428e49482b8c1916859fbe80564670fa51f8ff Mon Sep 17 00:00:00 2001 From: Morris Jobke Date: Thu, 6 Aug 2020 20:10:25 +0200 Subject: [PATCH 4/4] Add config option to enable multibucket preview distribution Signed-off-by: Morris Jobke --- config/config.sample.php | 17 +++++++++ .../ObjectStorePreviewCacheMountProvider.php | 3 ++ lib/private/Preview/Storage/Root.php | 18 +++++----- ...jectStorePreviewCacheMountProviderTest.php | 35 +++++++++++-------- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/config/config.sample.php b/config/config.sample.php index e95f2535af8..862c5ec1a1d 100644 --- a/config/config.sample.php +++ b/config/config.sample.php @@ -1354,6 +1354,23 @@ $CONFIG = [ ], ], +/** + * If this is set to true and a multibucket object store is configured then + * newly created previews are put into 256 dedicated buckets. + * + * Those buckets are named like the mulibucket version but with the postfix + * ``-preview-NUMBER`` where NUMBER is between 0 and 255. + * + * Keep in mind that only previews of files are put in there that don't have + * some already. Otherwise the old bucket will be used. + * + * To migrate existing previews to this new multibucket distribution of previews + * use the occ command ``preview:repair``. For now this will only migrate + * previews that were generated before Nextcloud 19 in the flat + * ``appdata_INSTANCEID/previews/FILEID`` folder structure. + */ +'objectstore.multibucket.preview-distribution' => false, + /** * Sharing diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php index dba1dfc28e5..9bbb744bbcc 100644 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php @@ -55,6 +55,9 @@ class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { return []; } + if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { + return []; + } $instanceId = $this->config->getSystemValueString('instanceid', ''); $mountPoints = []; diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php index 37ae1758121..a284b037b35 100644 --- a/lib/private/Preview/Storage/Root.php +++ b/lib/private/Preview/Storage/Root.php @@ -32,23 +32,23 @@ use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFolder; class Root extends AppData { + private $isMultibucketPreviewDistributionEnabled = false; public function __construct(IRootFolder $rootFolder, SystemConfig $systemConfig) { parent::__construct($rootFolder, $systemConfig, 'preview'); + + $this->isMultibucketPreviewDistributionEnabled = $systemConfig->getValue('objectstore.multibucket.preview-distribution', false) === true; } public function getFolder(string $name): ISimpleFolder { $internalFolder = $this->getInternalFolder($name); - try { - return parent::getFolder('old-multibucket/' . $internalFolder); - } catch (NotFoundException $e) { - // not in multibucket fallback #1 - } - try { - return parent::getFolder('old-multibucket/' . $name); - } catch (NotFoundException $e) { - // not in multibucket fallback #2 + if ($this->isMultibucketPreviewDistributionEnabled) { + try { + return parent::getFolder('old-multibucket/' . $internalFolder); + } catch (NotFoundException $e) { + // not in multibucket fallback + } } try { diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php index 2da07393f40..400808d7cd5 100644 --- a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php +++ b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php @@ -71,22 +71,29 @@ class ObjectStorePreviewCacheMountProviderTest extends \Test\TestCase { } public function testMultibucketObjectStorage() { + $objectstoreConfig = [ + 'class' => S3::class, + 'arguments' => [ + 'bucket' => 'abc', + 'num_buckets' => 64, + 'key' => 'KEY', + 'secret' => 'SECRET', + 'hostname' => 'IP', + 'port' => 'PORT', + 'use_ssl' => false, + 'use_path_style' => true, + ], + ]; $this->config->expects($this->any()) ->method('getSystemValue') - ->with('objectstore_multibucket') - ->willReturn([ - 'class' => S3::class, - 'arguments' => [ - 'bucket' => 'abc', - 'num_buckets' => 64, - 'key' => 'KEY', - 'secret' => 'SECRET', - 'hostname' => 'IP', - 'port' => 'PORT', - 'use_ssl' => false, - 'use_path_style' => true, - ], - ]); + ->willReturnCallback(function ($config) use ($objectstoreConfig) { + if ($config === 'objectstore_multibucket') { + return $objectstoreConfig; + } elseif ($config === 'objectstore.multibucket.preview-distribution') { + return true; + } + return null; + }); $this->config->expects($this->once()) ->method('getSystemValueString') ->with('instanceid')