Merge pull request #5442 from owncloud/extstorage-ignoreduplicateinserts

Ignore duplicate inserts in file cache and mime type

lib/private/files/cache/cache.php

@@ -69,9 +69,15 @@ class Cache {
 		}
 		
 		if (!isset(self::$mimetypeIds[$mime])) {
-			$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
-			self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
-			self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			try{
+				$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
+				self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
+				self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			}
+			catch (\Doctrine\DBAL\DBALException $e){
+				\OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
+				return -1;
+			}
 		} 
 				
 		return self::$mimetypeIds[$mime];
@@ -84,8 +90,8 @@ class Cache {
 
 		return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
 	}
-	
-	protected function loadMimetypes(){
+
+	public function loadMimetypes(){
 			$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
 			if ($result) {
 				while ($row = $result->fetchRow()) {

lib/private/files/cache/scanner.php

@@ -190,24 +190,34 @@ class Scanner extends BasicEmitter {
 		}
 		$newChildren = array();
 		if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
+			$exceptionOccurred = false;
 			\OC_DB::beginTransaction();
 			if (is_resource($dh)) {
 				while (($file = readdir($dh)) !== false) {
 					$child = ($path) ? $path . '/' . $file : $file;
 					if (!Filesystem::isIgnoredDir($file)) {
 						$newChildren[] = $file;
-						$data = $this->scanFile($child, $reuse, true);
-						if ($data) {
-							if ($data['size'] === -1) {
-								if ($recursive === self::SCAN_RECURSIVE) {
-									$childQueue[] = $child;
-								} else {
-									$size = -1;
+						try {
+							$data = $this->scanFile($child, $reuse, true);
+							if ($data) {
+								if ($data['size'] === -1) {
+									if ($recursive === self::SCAN_RECURSIVE) {
+										$childQueue[] = $child;
+									} else {
+										$size = -1;
+									}
+								} else if ($size !== -1) {
+									$size += $data['size'];
 								}
-							} else if ($size !== -1) {
-								$size += $data['size'];
 							}
 						}
+						catch (\Doctrine\DBAL\DBALException $ex){
+							// might happen if inserting duplicate while a scanning
+							// process is running in parallel
+							// log and ignore
+							\OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
+							$exceptionOccurred = true;
+						}
 					}
 				}
 			}
@@ -217,6 +227,14 @@ class Scanner extends BasicEmitter {
 				$this->cache->remove($child);
 			}
 			\OC_DB::commit();
+			if ($exceptionOccurred){
+				// It might happen that the parallel scan process has already
+				// inserted mimetypes but those weren't available yet inside the transaction
+				// To make sure to have the updated mime types in such cases,
+				// we reload them here
+				$this->cache->loadMimetypes();
+			}
+
 			foreach ($childQueue as $child) {
 				$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
 				if ($childSize === -1) {