The background job will now be executed in chunks of 500 users all 10 minutes.remotes/origin/share-copy-source-mounts
Lukas Reschke
10 years ago
parent
d305412a35
commit
391bc49dab
@ -1,94 +0,0 @@ |
||||
<?php |
||||
/** |
||||
* @author Bart Visscher <bartv@thisnet.nl> |
||||
* @author Jörn Friedrich Dreyer <jfd@butonic.de> |
||||
* @author Lukas Reschke <lukas@owncloud.com> |
||||
* @author Robin Appelman <icewind@owncloud.com> |
||||
* @author Vincent Petry <pvince81@owncloud.com> |
||||
* |
||||
* @copyright Copyright (c) 2015, ownCloud, Inc. |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
set_time_limit(0); //scanning can take ages |
||||
|
||||
\OCP\JSON::checkLoggedIn(); |
||||
\OCP\JSON::callCheck(); |
||||
|
||||
\OC::$server->getSession()->close(); |
||||
|
||||
$force = (isset($_GET['force']) and ($_GET['force'] === 'true')); |
||||
$dir = isset($_GET['dir']) ? (string)$_GET['dir'] : ''; |
||||
if (isset($_GET['users'])) { |
||||
\OCP\JSON::checkAdminUser(); |
||||
if ($_GET['users'] === 'all') { |
||||
$users = OC_User::getUsers(); |
||||
} else { |
||||
$users = json_decode($_GET['users']); |
||||
} |
||||
} else { |
||||
$users = array(OC_User::getUser()); |
||||
} |
||||
|
||||
$eventSource = \OC::$server->createEventSource(); |
||||
$listener = new ScanListener($eventSource); |
||||
|
||||
foreach ($users as $user) { |
||||
$eventSource->send('user', $user); |
||||
$scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection(), \OC::$server->getLogger()); |
||||
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file')); |
||||
try { |
||||
if ($force) { |
||||
$scanner->scan($dir); |
||||
} else { |
||||
$scanner->backgroundScan($dir); |
||||
} |
||||
} catch (\Exception $e) { |
||||
$eventSource->send('error', get_class($e) . ': ' . $e->getMessage()); |
||||
} |
||||
} |
||||
|
||||
$eventSource->send('done', $listener->getCount()); |
||||
$eventSource->close(); |
||||
|
||||
class ScanListener { |
||||
|
||||
private $fileCount = 0; |
||||
private $lastCount = 0; |
||||
|
||||
/** |
||||
* @var \OCP\IEventSource event source to pass events to |
||||
*/ |
||||
private $eventSource; |
||||
|
||||
/** |
||||
* @param \OCP\IEventSource $eventSource |
||||
*/ |
||||
public function __construct($eventSource) { |
||||
$this->eventSource = $eventSource; |
||||
} |
||||
|
||||
public function file() { |
||||
$this->fileCount++; |
||||
if ($this->fileCount > $this->lastCount + 20) { //send a count update every 20 files |
||||
$this->lastCount = $this->fileCount; |
||||
$this->eventSource->send('count', $this->fileCount); |
||||
} |
||||
} |
||||
|
||||
public function getCount() { |
||||
return $this->fileCount; |
||||
} |
||||
} |
||||
@ -0,0 +1,26 @@ |
||||
<?php |
||||
/** |
||||
* @author Lukas Reschke <lukas@owncloud.com> |
||||
* |
||||
* @copyright Copyright (c) 2015, ownCloud, Inc. |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
|
||||
// Cron job for scanning user storages |
||||
$jobList = \OC::$server->getJobList(); |
||||
$job = 'OCA\Files\BackgroundJob\ScanFiles'; |
||||
\OC::$server->getJobList()->add($job); |
||||
|
||||
@ -0,0 +1,114 @@ |
||||
<?php |
||||
/** |
||||
* @author Lukas Reschke <lukas@owncloud.com> |
||||
* |
||||
* @copyright Copyright (c) 2015, ownCloud, Inc. |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
|
||||
namespace OCA\Files\BackgroundJob; |
||||
|
||||
use OC\Files\Utils\Scanner; |
||||
use OCP\IConfig; |
||||
use OCP\IDBConnection; |
||||
use OCP\ILogger; |
||||
use OCP\IUser; |
||||
use OCP\IUserManager; |
||||
|
||||
/** |
||||
* Class ScanFiles is a background job used to run the file scanner over the user |
||||
* accounts to ensure integrity of the file cache. |
||||
* |
||||
* @package OCA\Files\BackgroundJob |
||||
*/ |
||||
class ScanFiles extends \OC\BackgroundJob\TimedJob { |
||||
/** @var IConfig */ |
||||
private $config; |
||||
/** @var IUserManager */ |
||||
private $userManager; |
||||
/** @var IDBConnection */ |
||||
private $dbConnection; |
||||
/** @var ILogger */ |
||||
private $logger; |
||||
/** Amount of users that should get scanned per execution */ |
||||
const USERS_PER_SESSION = 500; |
||||
|
||||
/** |
||||
* @param IConfig|null $config |
||||
* @param IUserManager|null $userManager |
||||
* @param IDBConnection|null $dbConnection |
||||
* @param ILogger|null $logger |
||||
*/ |
||||
public function __construct(IConfig $config = null, |
||||
IUserManager $userManager = null, |
||||
IDBConnection $dbConnection = null, |
||||
ILogger $logger = null) { |
||||
// Run once per 10 minutes |
||||
$this->setInterval(60 * 10); |
||||
|
||||
if (is_null($userManager) || is_null($config)) { |
||||
$this->fixDIForJobs(); |
||||
} else { |
||||
$this->config = $config; |
||||
$this->userManager = $userManager; |
||||
$this->logger = $logger; |
||||
} |
||||
} |
||||
|
||||
protected function fixDIForJobs() { |
||||
$this->config = \OC::$server->getConfig(); |
||||
$this->userManager = \OC::$server->getUserManager(); |
||||
$this->logger = \OC::$server->getLogger(); |
||||
} |
||||
|
||||
/** |
||||
* @param IUser $user |
||||
*/ |
||||
protected function runScanner(IUser $user) { |
||||
try { |
||||
$scanner = new Scanner( |
||||
$user->getUID(), |
||||
$this->dbConnection, |
||||
$this->logger |
||||
); |
||||
$scanner->backgroundScan(''); |
||||
} catch (\Exception $e) { |
||||
$this->logger->logException($e, ['app' => 'files']); |
||||
} |
||||
\OC_Util::tearDownFS(); |
||||
} |
||||
|
||||
/** |
||||
* @param $argument |
||||
* @throws \Exception |
||||
*/ |
||||
protected function run($argument) { |
||||
$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0); |
||||
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset); |
||||
if (!count($users)) { |
||||
// No users found, reset offset and retry |
||||
$offset = 0; |
||||
$users = $this->userManager->search('', self::USERS_PER_SESSION); |
||||
} |
||||
|
||||
$offset += self::USERS_PER_SESSION; |
||||
$this->config->setAppValue('files', 'cronjob_scan_files', $offset); |
||||
|
||||
foreach ($users as $user) { |
||||
$this->runScanner($user); |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,134 @@ |
||||
<?php |
||||
/** |
||||
* @author Lukas Reschke <lukas@owncloud.com> |
||||
* |
||||
* @copyright Copyright (c) 2015, ownCloud, Inc. |
||||
* @license AGPL-3.0 |
||||
* |
||||
* This code is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU Affero General Public License, version 3, |
||||
* as published by the Free Software Foundation. |
||||
* |
||||
* This program is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU Affero General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU Affero General Public License, version 3, |
||||
* along with this program. If not, see <http://www.gnu.org/licenses/> |
||||
* |
||||
*/ |
||||
namespace OCA\Files\Tests\BackgroundJob; |
||||
|
||||
use Test\TestCase; |
||||
use OCP\IConfig; |
||||
use OCP\IUserManager; |
||||
use OCA\Files\BackgroundJob\ScanFiles; |
||||
use OCP\ILogger; |
||||
|
||||
/** |
||||
* Class ScanFilesTest |
||||
* |
||||
* @package OCA\Files\Tests\BackgroundJob |
||||
*/ |
||||
class ScanFilesTest extends TestCase { |
||||
/** @var IConfig */ |
||||
private $config; |
||||
/** @var IUserManager */ |
||||
private $userManager; |
||||
/** @var ScanFiles */ |
||||
private $scanFiles; |
||||
|
||||
public function setUp() { |
||||
parent::setUp(); |
||||
|
||||
$this->config = $this->getMock('\OCP\IConfig'); |
||||
$this->userManager = $this->getMock('\OCP\IUserManager'); |
||||
|
||||
$this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles') |
||||
->setConstructorArgs([ |
||||
$this->config, |
||||
$this->userManager, |
||||
]) |
||||
->setMethods(['runScanner']) |
||||
->getMock(); |
||||
} |
||||
|
||||
public function testRunWithoutUsers() { |
||||
$this->config |
||||
->expects($this->at(0)) |
||||
->method('getAppValue') |
||||
->with('files', 'cronjob_scan_files', 0) |
||||
->will($this->returnValue(50)); |
||||
$this->userManager |
||||
->expects($this->at(0)) |
||||
->method('search') |
||||
->with('', 500, 50) |
||||
->will($this->returnValue([])); |
||||
$this->userManager |
||||
->expects($this->at(1)) |
||||
->method('search') |
||||
->with('', 500) |
||||
->will($this->returnValue([])); |
||||
$this->config |
||||
->expects($this->at(1)) |
||||
->method('setAppValue') |
||||
->with('files', 'cronjob_scan_files', 500); |
||||
|
||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); |
||||
} |
||||
|
||||
public function testRunWithUsers() { |
||||
$fakeUser = $this->getMock('\OCP\IUser'); |
||||
$this->config |
||||
->expects($this->at(0)) |
||||
->method('getAppValue') |
||||
->with('files', 'cronjob_scan_files', 0) |
||||
->will($this->returnValue(50)); |
||||
$this->userManager |
||||
->expects($this->at(0)) |
||||
->method('search') |
||||
->with('', 500, 50) |
||||
->will($this->returnValue([ |
||||
$fakeUser |
||||
])); |
||||
$this->config |
||||
->expects($this->at(1)) |
||||
->method('setAppValue') |
||||
->with('files', 'cronjob_scan_files', 550); |
||||
$this->scanFiles |
||||
->expects($this->once()) |
||||
->method('runScanner') |
||||
->with($fakeUser); |
||||
|
||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); |
||||
} |
||||
|
||||
public function testRunWithUsersAndOffsetAtEndOfUserList() { |
||||
$this->config |
||||
->expects($this->at(0)) |
||||
->method('getAppValue') |
||||
->with('files', 'cronjob_scan_files', 0) |
||||
->will($this->returnValue(50)); |
||||
$this->userManager |
||||
->expects($this->at(0)) |
||||
->method('search') |
||||
->with('', 500, 50) |
||||
->will($this->returnValue([])); |
||||
$this->userManager |
||||
->expects($this->at(1)) |
||||
->method('search') |
||||
->with('', 500) |
||||
->will($this->returnValue([])); |
||||
$this->config |
||||
->expects($this->at(1)) |
||||
->method('setAppValue') |
||||
->with('files', 'cronjob_scan_files', 500); |
||||
$this->scanFiles |
||||
->expects($this->never()) |
||||
->method('runScanner'); |
||||
|
||||
$this->invokePrivate($this->scanFiles, 'run', [[]]); |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue