From 644755df663e8dc295e92700da208a28c13cbaab Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Mon, 8 Sep 2014 01:34:03 +0200 Subject: [PATCH 1/2] Use bigger transactions when doing explicit file system scans --- apps/files/ajax/scan.php | 2 +- apps/files/command/scan.php | 2 +- lib/private/files/cache/scanner.php | 23 +++++++++++++++++++++-- lib/private/files/utils/scanner.php | 12 +++++++++++- tests/lib/files/etagtest.php | 2 +- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/apps/files/ajax/scan.php b/apps/files/ajax/scan.php index 3ec7f9394b1..43760cc4196 100644 --- a/apps/files/ajax/scan.php +++ b/apps/files/ajax/scan.php @@ -20,7 +20,7 @@ $listener = new ScanListener($eventSource); foreach ($users as $user) { $eventSource->send('user', $user); - $scanner = new \OC\Files\Utils\Scanner($user); + $scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection()); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', array($listener, 'file')); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', array($listener, 'folder')); if ($force) { diff --git a/apps/files/command/scan.php b/apps/files/command/scan.php index 3412cf80dea..e4d719a8b4b 100644 --- a/apps/files/command/scan.php +++ b/apps/files/command/scan.php @@ -46,7 +46,7 @@ class Scan extends Command { } protected function scanFiles($user, OutputInterface $output) { - $scanner = new \OC\Files\Utils\Scanner($user); + $scanner = new \OC\Files\Utils\Scanner($user, \OC::$server->getDatabaseConnection()); $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) { $output->writeln("Scanning $path"); }); diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index 2d87871fd89..cca4f289c80 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -44,6 +44,11 @@ class Scanner extends BasicEmitter { */ protected $cacheActive; + /** + * @var bool $transactions whether to use transactions + */ + protected $useTransactions = true; + const SCAN_RECURSIVE = true; const SCAN_SHALLOW = false; @@ -57,6 +62,16 @@ class Scanner extends BasicEmitter { $this->cacheActive = !Config::getSystemValue('filesystem_cache_readonly', false); } + /** + * Whether to wrap the scanning of a folder in a database transaction + * On default transactions are used + * + * @param bool $useTransactions + */ + public function setUseTransactions($useTransactions) { + $this->useTransactions = $useTransactions; + } + /** * get all the metadata of a file or folder * * @@ -234,7 +249,9 @@ class Scanner extends BasicEmitter { $newChildren = array(); if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) { $exceptionOccurred = false; - \OC_DB::beginTransaction(); + if ($this->useTransactions) { + \OC_DB::beginTransaction(); + } if (is_resource($dh)) { while (($file = readdir($dh)) !== false) { $child = ($path) ? $path . '/' . $file : $file; @@ -266,7 +283,9 @@ class Scanner extends BasicEmitter { $child = ($path) ? $path . '/' . $childName : $childName; $this->removeFromCache($child); } - \OC_DB::commit(); + if ($this->useTransactions) { + \OC_DB::commit(); + } if ($exceptionOccurred) { // It might happen that the parallel scan process has already // inserted mimetypes but those weren't available yet inside the transaction diff --git a/lib/private/files/utils/scanner.php b/lib/private/files/utils/scanner.php index c2fabf51946..c7da4505af5 100644 --- a/lib/private/files/utils/scanner.php +++ b/lib/private/files/utils/scanner.php @@ -34,12 +34,19 @@ class Scanner extends PublicEmitter { */ protected $propagator; + /** + * @var \OCP\IDBConnection + */ + protected $db; + /** * @param string $user + * @param \OCP\IDBConnection $db */ - public function __construct($user) { + public function __construct($user, $db) { $this->user = $user; $this->propagator = new ChangePropagator(new View('')); + $this->db = $db; } /** @@ -121,8 +128,11 @@ class Scanner extends PublicEmitter { throw new ForbiddenException(); } $scanner = $storage->getScanner(); + $scanner->useTransactions(false); $this->attachListener($mount); + $this->db->beginTransaction(); $scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->db->commit(); } $this->propagator->propagateChanges(time()); } diff --git a/tests/lib/files/etagtest.php b/tests/lib/files/etagtest.php index af9f66835f0..b5dec107e79 100644 --- a/tests/lib/files/etagtest.php +++ b/tests/lib/files/etagtest.php @@ -62,7 +62,7 @@ class EtagTest extends \PHPUnit_Framework_TestCase { $files = array('/foo.txt', '/folder/bar.txt', '/folder/subfolder', '/folder/subfolder/qwerty.txt'); $originalEtags = $this->getEtags($files); - $scanner = new \OC\Files\Utils\Scanner($user1); + $scanner = new \OC\Files\Utils\Scanner($user1, \OC::$server->getDatabaseConnection()); $scanner->backgroundScan('/'); $newEtags = $this->getEtags($files); From fa718d2e2cccb8e48b89044166a80b82b91288fa Mon Sep 17 00:00:00 2001 From: Lukas Reschke Date: Mon, 8 Sep 2014 16:34:03 +0200 Subject: [PATCH 2/2] Fix typo --- lib/private/files/cache/scanner.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/private/files/cache/scanner.php b/lib/private/files/cache/scanner.php index cca4f289c80..dfba2d920a0 100644 --- a/lib/private/files/cache/scanner.php +++ b/lib/private/files/cache/scanner.php @@ -45,7 +45,7 @@ class Scanner extends BasicEmitter { protected $cacheActive; /** - * @var bool $transactions whether to use transactions + * @var bool $useTransactions whether to use transactions */ protected $useTransactions = true;