Merge pull request #22063 from nextcloud/enh/22033/multibucket-appdata-previews
Distribute preview folders in appdata in multibucket setup to multiple bucketspull/22139/head
Morris Jobke
5 years ago
committed by
GitHub
commit
eb2b1bec85
@ -0,0 +1,150 @@ |
||||
<?php |
||||
|
||||
declare(strict_types=1); |
||||
/** |
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @author Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
|
||||
namespace OC\Files\Mount; |
||||
|
||||
use OC\Files\ObjectStore\AppdataPreviewObjectStoreStorage; |
||||
use OC\Files\ObjectStore\ObjectStoreStorage; |
||||
use OC\Files\Storage\Wrapper\Jail; |
||||
use OCP\Files\Config\IRootMountProvider; |
||||
use OCP\Files\Storage\IStorageFactory; |
||||
use OCP\IConfig; |
||||
use OCP\ILogger; |
||||
|
||||
/** |
||||
* Mount provider for object store app data folder for previews |
||||
*/ |
||||
class ObjectStorePreviewCacheMountProvider implements IRootMountProvider { |
||||
/** @var ILogger */ |
||||
private $logger; |
||||
/** @var IConfig */ |
||||
private $config; |
||||
|
||||
public function __construct(ILogger $logger, IConfig $config) { |
||||
$this->logger = $logger; |
||||
$this->config = $config; |
||||
} |
||||
|
||||
/** |
||||
* @return MountPoint[] |
||||
* @throws \Exception |
||||
*/ |
||||
public function getRootMounts(IStorageFactory $loader): array { |
||||
if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { |
||||
return []; |
||||
} |
||||
if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { |
||||
return []; |
||||
} |
||||
|
||||
$instanceId = $this->config->getSystemValueString('instanceid', ''); |
||||
$mountPoints = []; |
||||
$directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; |
||||
$i = 0; |
||||
foreach ($directoryRange as $parent) { |
||||
foreach ($directoryRange as $child) { |
||||
$mountPoints[] = new MountPoint( |
||||
AppdataPreviewObjectStoreStorage::class, |
||||
'/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, |
||||
$this->getMultiBucketObjectStore($i), |
||||
$loader |
||||
); |
||||
$i++; |
||||
} |
||||
} |
||||
|
||||
$rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); |
||||
$fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); |
||||
$fakeRootStorageJail = new Jail([ |
||||
'storage' => $fakeRootStorage, |
||||
'root' => '/appdata_' . $instanceId . '/preview', |
||||
]); |
||||
|
||||
// add a fallback location to be able to fetch existing previews from the old bucket |
||||
$mountPoints[] = new MountPoint( |
||||
$fakeRootStorageJail, |
||||
'/appdata_' . $instanceId . '/preview/old-multibucket', |
||||
null, |
||||
$loader |
||||
); |
||||
|
||||
return $mountPoints; |
||||
} |
||||
|
||||
protected function getMultiBucketObjectStore(int $number): array { |
||||
$config = $this->config->getSystemValue('objectstore_multibucket'); |
||||
|
||||
// sanity checks |
||||
if (empty($config['class'])) { |
||||
$this->logger->error('No class given for objectstore', ['app' => 'files']); |
||||
} |
||||
if (!isset($config['arguments'])) { |
||||
$config['arguments'] = []; |
||||
} |
||||
|
||||
/* |
||||
* Use any provided bucket argument as prefix |
||||
* and add the mapping from parent/child => bucket |
||||
*/ |
||||
if (!isset($config['arguments']['bucket'])) { |
||||
$config['arguments']['bucket'] = ''; |
||||
} |
||||
|
||||
$config['arguments']['bucket'] .= "-preview-$number"; |
||||
|
||||
// instantiate object store implementation |
||||
$config['arguments']['objectstore'] = new $config['class']($config['arguments']); |
||||
|
||||
$config['arguments']['internal-id'] = $number; |
||||
|
||||
return $config['arguments']; |
||||
} |
||||
|
||||
protected function getMultiBucketObjectStoreForRoot(): array { |
||||
$config = $this->config->getSystemValue('objectstore_multibucket'); |
||||
|
||||
// sanity checks |
||||
if (empty($config['class'])) { |
||||
$this->logger->error('No class given for objectstore', ['app' => 'files']); |
||||
} |
||||
if (!isset($config['arguments'])) { |
||||
$config['arguments'] = []; |
||||
} |
||||
|
||||
/* |
||||
* Use any provided bucket argument as prefix |
||||
* and add the mapping from parent/child => bucket |
||||
*/ |
||||
if (!isset($config['arguments']['bucket'])) { |
||||
$config['arguments']['bucket'] = ''; |
||||
} |
||||
$config['arguments']['bucket'] .= '0'; |
||||
|
||||
// instantiate object store implementation |
||||
$config['arguments']['objectstore'] = new $config['class']($config['arguments']); |
||||
|
||||
return $config['arguments']; |
||||
} |
||||
} |
||||
@ -0,0 +1,44 @@ |
||||
<?php |
||||
|
||||
declare(strict_types=1); |
||||
/** |
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @author Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
|
||||
namespace OC\Files\ObjectStore; |
||||
|
||||
class AppdataPreviewObjectStoreStorage extends ObjectStoreStorage { |
||||
|
||||
/** @var string */ |
||||
private $internalId; |
||||
|
||||
public function __construct($params) { |
||||
if (!isset($params['internal-id'])) { |
||||
throw new \Exception('missing id in parameters'); |
||||
} |
||||
$this->internalId = (string)$params['internal-id']; |
||||
parent::__construct($params); |
||||
} |
||||
|
||||
public function getId() { |
||||
return 'object::appdata::preview:' . $this->internalId; |
||||
} |
||||
} |
||||
@ -0,0 +1,41 @@ |
||||
<?php |
||||
|
||||
declare(strict_types=1); |
||||
/** |
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @author Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
|
||||
namespace OCP\Files\Config; |
||||
|
||||
use OCP\Files\Storage\IStorageFactory; |
||||
|
||||
/** |
||||
* @since 20.0.0 |
||||
*/ |
||||
interface IRootMountProvider { |
||||
/** |
||||
* Get all root mountpoints |
||||
* |
||||
* @return \OCP\Files\Mount\IMountPoint[] |
||||
* @since 20.0.0 |
||||
*/ |
||||
public function getRootMounts(IStorageFactory $loader): array; |
||||
} |
||||
@ -0,0 +1,113 @@ |
||||
<?php |
||||
|
||||
declare(strict_types=1); |
||||
/** |
||||
* @copyright Copyright (c) 2020, Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @author Morris Jobke <hey@morrisjobke.de> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
|
||||
namespace Test\Files\Mount; |
||||
|
||||
use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; |
||||
use OC\Files\ObjectStore\S3; |
||||
use OC\Files\Storage\StorageFactory; |
||||
use OCP\Files\Storage\IStorageFactory; |
||||
use OCP\IConfig; |
||||
use OCP\ILogger; |
||||
use PHPUnit\Framework\MockObject\MockObject; |
||||
|
||||
/** |
||||
* @group DB |
||||
* |
||||
* The DB permission is needed for the fake root storage initialization |
||||
*/ |
||||
class ObjectStorePreviewCacheMountProviderTest extends \Test\TestCase { |
||||
|
||||
/** @var ObjectStorePreviewCacheMountProvider */ |
||||
protected $provider; |
||||
|
||||
/** @var ILogger|MockObject */ |
||||
protected $logger; |
||||
/** @var IConfig|MockObject */ |
||||
protected $config; |
||||
/** @var IStorageFactory|MockObject */ |
||||
protected $loader; |
||||
|
||||
|
||||
protected function setUp(): void { |
||||
parent::setUp(); |
||||
|
||||
$this->logger = $this->createMock(ILogger::class); |
||||
$this->config = $this->createMock(IConfig::class); |
||||
$this->loader = $this->createMock(StorageFactory::class); |
||||
|
||||
$this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config); |
||||
} |
||||
|
||||
public function testNoMultibucketObjectStorage() { |
||||
$this->config->expects($this->once()) |
||||
->method('getSystemValue') |
||||
->with('objectstore_multibucket') |
||||
->willReturn(null); |
||||
|
||||
$this->assertEquals([], $this->provider->getRootMounts($this->loader)); |
||||
} |
||||
|
||||
public function testMultibucketObjectStorage() { |
||||
$objectstoreConfig = [ |
||||
'class' => S3::class, |
||||
'arguments' => [ |
||||
'bucket' => 'abc', |
||||
'num_buckets' => 64, |
||||
'key' => 'KEY', |
||||
'secret' => 'SECRET', |
||||
'hostname' => 'IP', |
||||
'port' => 'PORT', |
||||
'use_ssl' => false, |
||||
'use_path_style' => true, |
||||
], |
||||
]; |
||||
$this->config->expects($this->any()) |
||||
->method('getSystemValue') |
||||
->willReturnCallback(function ($config) use ($objectstoreConfig) { |
||||
if ($config === 'objectstore_multibucket') { |
||||
return $objectstoreConfig; |
||||
} elseif ($config === 'objectstore.multibucket.preview-distribution') { |
||||
return true; |
||||
} |
||||
return null; |
||||
}); |
||||
$this->config->expects($this->once()) |
||||
->method('getSystemValueString') |
||||
->with('instanceid') |
||||
->willReturn('INSTANCEID'); |
||||
|
||||
$mounts = $this->provider->getRootMounts($this->loader); |
||||
|
||||
// 256 mounts for the subfolders and 1 for the fake root |
||||
$this->assertCount(257, $mounts); |
||||
|
||||
// do some sanity checks if they have correct mount point paths |
||||
$this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint()); |
||||
$this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint()); |
||||
// also test the path of the fake bucket |
||||
$this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue