Merge pull request #36788 from nextcloud/revert-36589-enh/perf-remove-icache
Revert "fix(performance): Do not set up filesystem on every call"pull/36792/head
Anna
3 years ago
committed by
GitHub
commit
13ef475a0a
@ -1,62 +0,0 @@ |
||||
<?php |
||||
declare(strict_types=1); |
||||
|
||||
/** |
||||
* @copyright 2023 Anna Larch <anna.larch@gmx.net> |
||||
* |
||||
* @author Anna Larch <anna.larch@gmx.net> |
||||
* |
||||
* @license GNU AGPL version 3 or any later version |
||||
* |
||||
* This program is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License as |
||||
* published by the Free Software Foundation, either version 3 of the |
||||
* License, or (at your option) any later version. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. |
||||
* |
||||
*/ |
||||
namespace OCA\Files\BackgroundJob; |
||||
|
||||
use OC\Cache\File; |
||||
use OCP\AppFramework\Utility\ITimeFactory; |
||||
use OCP\BackgroundJob\Job; |
||||
use OCP\BackgroundJob\TimedJob; |
||||
use OCP\Files\IRootFolder; |
||||
use OCP\IUser; |
||||
use OCP\IUserManager; |
||||
use Psr\Log\LoggerInterface; |
||||
|
||||
class FileChunkCleanupJob extends TimedJob { |
||||
private IUserManager $userManager; |
||||
private IRootFolder $rootFolder; |
||||
private LoggerInterface $logger; |
||||
|
||||
public function __construct(IUserManager $userManager, IRootFolder $rootFolder, LoggerInterface $logger, ITimeFactory $timeFactory) { |
||||
parent::__construct($timeFactory); |
||||
$this->setInterval(3600*24); |
||||
$this->setTimeSensitivity(Job::TIME_INSENSITIVE); |
||||
$this->userManager = $userManager; |
||||
$this->rootFolder = $rootFolder; |
||||
$this->logger = $logger; |
||||
} |
||||
|
||||
/** |
||||
* This job cleans up all backups except the latest 3 from the updaters backup directory |
||||
*/ |
||||
public function run($argument): void { |
||||
$this->userManager->callForSeenUsers(function (IUser $user): void { |
||||
$this->logger->debug('Running chunk cleanup job for user '. $user->getUID()); |
||||
$fileCache = new File(); |
||||
$fileCache->setUpStorage($user->getUID()); |
||||
$fileCache->gc(); |
||||
$this->logger->debug('Finished running chunk cleanup job for user '. $user->getUID()); |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,184 @@ |
||||
<?php |
||||
/** |
||||
* @copyright Copyright (c) 2016, ownCloud, Inc. |
||||
* |
||||
* @author Bart Visscher <bartv@thisnet.nl> |
||||
* @author Christoph Wurst <christoph@winzerhof-wurst.at> |
||||
* @author Felix Moeller <mail@felixmoeller.de> |
||||
* @author Jörn Friedrich Dreyer <jfd@butonic.de> |
||||
* @author Morris Jobke <hey@morrisjobke.de> |
||||
* @author Robin Appelman <robin@icewind.nl> |
||||
* @author Roeland Jago Douma <roeland@famdouma.nl> |
||||
* @author Thomas Müller <thomas.mueller@tmit.eu> |
||||
* @author Thomas Tanghus <thomas@tanghus.net> |
||||
* @author Vincent Petry <vincent@nextcloud.com> |
||||
* |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
class OC_FileChunking { |
||||
protected $info; |
||||
protected $cache; |
||||
|
||||
/** |
||||
* TTL of chunks |
||||
* |
||||
* @var int |
||||
*/ |
||||
protected $ttl; |
||||
|
||||
public static function decodeName($name) { |
||||
preg_match('/(?P<name>.*)-chunking-(?P<transferid>\d+)-(?P<chunkcount>\d+)-(?P<index>\d+)/', $name, $matches); |
||||
return $matches; |
||||
} |
||||
|
||||
/** |
||||
* @param string[] $info |
||||
*/ |
||||
public function __construct($info) { |
||||
$this->info = $info; |
||||
$this->ttl = \OC::$server->getConfig()->getSystemValue('cache_chunk_gc_ttl', 86400); |
||||
} |
||||
|
||||
public function getPrefix() { |
||||
$name = $this->info['name']; |
||||
$transferid = $this->info['transferid']; |
||||
|
||||
return $name.'-chunking-'.$transferid.'-'; |
||||
} |
||||
|
||||
protected function getCache() { |
||||
if (!isset($this->cache)) { |
||||
$this->cache = new \OC\Cache\File(); |
||||
} |
||||
return $this->cache; |
||||
} |
||||
|
||||
/** |
||||
* Stores the given $data under the given $key - the number of stored bytes is returned |
||||
* |
||||
* @param string $index |
||||
* @param resource $data |
||||
* @return int |
||||
*/ |
||||
public function store($index, $data) { |
||||
$cache = $this->getCache(); |
||||
$name = $this->getPrefix().$index; |
||||
$cache->set($name, $data, $this->ttl); |
||||
|
||||
return $cache->size($name); |
||||
} |
||||
|
||||
public function isComplete() { |
||||
$prefix = $this->getPrefix(); |
||||
$cache = $this->getCache(); |
||||
$chunkcount = (int)$this->info['chunkcount']; |
||||
|
||||
for ($i = ($chunkcount - 1); $i >= 0; $i--) { |
||||
if (!$cache->hasKey($prefix.$i)) { |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
return true; |
||||
} |
||||
|
||||
/** |
||||
* Assembles the chunks into the file specified by the path. |
||||
* Chunks are deleted afterwards. |
||||
* |
||||
* @param resource $f target path |
||||
* |
||||
* @return integer assembled file size |
||||
* |
||||
* @throws \OC\InsufficientStorageException when file could not be fully |
||||
* assembled due to lack of free space |
||||
*/ |
||||
public function assemble($f) { |
||||
$cache = $this->getCache(); |
||||
$prefix = $this->getPrefix(); |
||||
$count = 0; |
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) { |
||||
$chunk = $cache->get($prefix.$i); |
||||
// remove after reading to directly save space |
||||
$cache->remove($prefix.$i); |
||||
$count += fwrite($f, $chunk); |
||||
// let php release the memory to work around memory exhausted error with php 5.6 |
||||
$chunk = null; |
||||
} |
||||
|
||||
return $count; |
||||
} |
||||
|
||||
/** |
||||
* Returns the size of the chunks already present |
||||
* @return integer size in bytes |
||||
*/ |
||||
public function getCurrentSize() { |
||||
$cache = $this->getCache(); |
||||
$prefix = $this->getPrefix(); |
||||
$total = 0; |
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) { |
||||
$total += $cache->size($prefix.$i); |
||||
} |
||||
return $total; |
||||
} |
||||
|
||||
/** |
||||
* Removes all chunks which belong to this transmission |
||||
*/ |
||||
public function cleanup() { |
||||
$cache = $this->getCache(); |
||||
$prefix = $this->getPrefix(); |
||||
for ($i = 0; $i < $this->info['chunkcount']; $i++) { |
||||
$cache->remove($prefix.$i); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Removes one specific chunk |
||||
* @param string $index |
||||
*/ |
||||
public function remove($index) { |
||||
$cache = $this->getCache(); |
||||
$prefix = $this->getPrefix(); |
||||
$cache->remove($prefix.$index); |
||||
} |
||||
|
||||
/** |
||||
* Assembles the chunks into the file specified by the path. |
||||
* Also triggers the relevant hooks and proxies. |
||||
* |
||||
* @param \OC\Files\Storage\Storage $storage storage |
||||
* @param string $path target path relative to the storage |
||||
* @return bool true on success or false if file could not be created |
||||
* |
||||
* @throws \OC\ServerNotAvailableException |
||||
*/ |
||||
public function file_assemble($storage, $path) { |
||||
// use file_put_contents as method because that best matches what this function does |
||||
if (\OC\Files\Filesystem::isValidPath($path)) { |
||||
$target = $storage->fopen($path, 'w'); |
||||
if ($target) { |
||||
$count = $this->assemble($target); |
||||
fclose($target); |
||||
return $count > 0; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
} |
||||
@ -0,0 +1,72 @@ |
||||
<?php |
||||
/** |
||||
* @author Roeland Jago Douma <rullzer@owncloud.com> |
||||
* |
||||
* @copyright Copyright (c) 2016, ownCloud, Inc. |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
|
||||
namespace Test; |
||||
|
||||
use OCP\ICache; |
||||
|
||||
class FileChunkingTest extends \Test\TestCase { |
||||
public function dataIsComplete() { |
||||
return [ |
||||
[1, [], false], |
||||
[1, [0], true], |
||||
[2, [], false], |
||||
[2, [0], false], |
||||
[2, [1], false], |
||||
[2, [0,1], true], |
||||
[10, [], false], |
||||
[10, [0,1,2,3,4,5,6,7,8], false], |
||||
[10, [1,2,3,4,5,6,7,8,9], false], |
||||
[10, [0,1,2,3,5,6,7,8,9], false], |
||||
[10, [0,1,2,3,4,5,6,7,8,9], true], |
||||
]; |
||||
} |
||||
|
||||
/** |
||||
* @dataProvider dataIsComplete |
||||
* @param $total |
||||
* @param array $present |
||||
* @param $expected |
||||
*/ |
||||
public function testIsComplete($total, array $present, $expected) { |
||||
$fileChunking = $this->getMockBuilder(\OC_FileChunking::class) |
||||
->setMethods(['getCache']) |
||||
->setConstructorArgs([[ |
||||
'name' => 'file', |
||||
'transferid' => '42', |
||||
'chunkcount' => $total, |
||||
]]) |
||||
->getMock(); |
||||
|
||||
$cache = $this->createMock(ICache::class); |
||||
|
||||
$cache->expects($this->atLeastOnce()) |
||||
->method('hasKey') |
||||
->willReturnCallback(function ($key) use ($present) { |
||||
$data = explode('-', $key); |
||||
return in_array($data[3], $present); |
||||
}); |
||||
|
||||
$fileChunking->method('getCache')->willReturn($cache); |
||||
|
||||
$this->assertEquals($expected, $fileChunking->isComplete()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue