OCC scanner commit in batches

lib/private/Files/Utils/Scanner.php

@@ -47,6 +47,8 @@ use OCP\ILogger;
  * @package OC\Files\Utils
  */
 class Scanner extends PublicEmitter {
+	const MAX_ENTRIES_TO_COMMIT = 10000;
+
 	/**
 	 * @var string $user
 	 */
@@ -62,6 +64,20 @@ class Scanner extends PublicEmitter {
 	 */
 	protected $logger;
 
+	/**
+	 * Whether to use a DB transaction
+	 *
+	 * @var bool
+	 */
+	protected $useTransaction;
+
+	/**
+	 * Number of entries scanned to commit
+	 *
+	 * @var int
+	 */
+	protected $entriesToCommit;
+
 	/**
 	 * @param string $user
 	 * @param \OCP\IDBConnection $db
@@ -71,6 +87,7 @@ class Scanner extends PublicEmitter {
 		$this->logger = $logger;
 		$this->user = $user;
 		$this->db = $db;
+		$this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider);
 	}
 
 	/**
@@ -200,22 +217,22 @@ class Scanner extends PublicEmitter {
 			$scanner = $storage->getScanner();
 			$scanner->setUseTransactions(false);
 			$this->attachListener($mount);
-			$isDbLocking = \OC::$server->getLockingProvider() instanceof DBLockingProvider;
 
 			$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 			$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 			$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) {
-				$this->triggerPropagator($storage, $path);
+				$this->postProcessEntry($storage, $path);
 			});
 
 			if (!$storage->file_exists($relativePath)) {
 				throw new NotFoundException($dir);
 			}
-			if (!$isDbLocking) {
+
+			if ($this->useTransaction) {
 				$this->db->beginTransaction();
 			}
 			try {
@@ -233,7 +250,7 @@ class Scanner extends PublicEmitter {
 				$this->logger->logException($e);
 				$this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]);
 			}
-			if (!$isDbLocking) {
+			if ($this->useTransaction) {
 				$this->db->commit();
 			}
 		}
@@ -242,5 +259,20 @@ class Scanner extends PublicEmitter {
 	private function triggerPropagator(IStorage $storage, $internalPath) {
 		$storage->getPropagator()->propagateChange($internalPath, time());
 	}
+
+	private function postProcessEntry(IStorage $storage, $internalPath) {
+		$this->triggerPropagator($storage, $internalPath);
+		$this->entriesToCommit++;
+		if ($this->useTransaction) {
+			$propagator = $storage->getPropagator();
+			if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) {
+				$this->entriesToCommit = 0;
+				$this->db->commit();
+				$propagator->commitBatch();
+				$this->db->beginTransaction();
+				$propagator->beginBatch();
+			}
+		}
+	}
 }