Merge pull request #7539 from owncloud/repair-legacystorageid

Added repair script to upgrade the legacy home storage ids

lib/private/db/connectionfactory.php

@@ -102,6 +102,11 @@ class ConnectionFactory {
 				/** @var $connection \OC\DB\Connection */
 				$connection->disableQueryStatementCaching();
 				break;
+			case 'oci':
+				// oracle seems to have issues with cached statements which have been closed
+				/** @var $connection \OC\DB\Connection */
+				$connection->disableQueryStatementCaching();
+				break;
 		}
 		return $connection;
 	}

lib/private/files/cache/storage.php

@@ -28,9 +28,7 @@ class Storage {
 		} else {
 			$this->storageId = $storage;
 		}
-		if (strlen($this->storageId) > 64) {
+		$this->storageId = self::adjustStorageId($this->storageId);
-			$this->storageId = md5($this->storageId);
-		}
 
 		$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
 		$result = \OC_DB::executeAudited($sql, array($this->storageId));
@@ -43,6 +41,19 @@ class Storage {
 		}
 	}
 
+	/**
+	 * Adjusts the storage id to use md5 if too long
+	 * @param string $storageId storage id
+	 * @return unchanged $storageId if its length is less than 64 characters,
+	 * else returns the md5 of $storageId
+	 */
+	public static function adjustStorageId($storageId) {
+		if (strlen($storageId) > 64) {
+			return md5($storageId);
+		}
+		return $storageId;
+	}
+
 	/**
 	 * @return string
 	 */
@@ -68,9 +79,7 @@ class Storage {
 	 * @return string|null
 	 */
 	public static function getNumericStorageId($storageId) {
-		if (strlen($storageId) > 64) {
+		$storageId = self::adjustStorageId($storageId);
-			$storageId = md5($storageId);
-		}
 
 		$sql = 'SELECT `numeric_id` FROM `*PREFIX*storages` WHERE `id` = ?';
 		$result = \OC_DB::executeAudited($sql, array($storageId));
@@ -95,9 +104,7 @@ class Storage {
 	 * @param string $storageId
 	 */
 	public static function remove($storageId) {
-		if (strlen($storageId) > 64) {
+		$storageId = self::adjustStorageId($storageId);
-			$storageId = md5($storageId);
-		}
 		$sql = 'DELETE FROM `*PREFIX*storages` WHERE `id` = ?';
 		\OC_DB::executeAudited($sql, array($storageId));
 

lib/private/repair.php

@@ -69,7 +69,8 @@ class Repair extends BasicEmitter {
 	 */
 	public static function getRepairSteps() {
 		return array(
-			new \OC\Repair\RepairMimeTypes()
+			new \OC\Repair\RepairMimeTypes(),
+			new \OC\Repair\RepairLegacyStorages(\OC::$server->getConfig(), \OC_DB::getConnection()),
 		);
 	}
 

lib/private/repair/repairlegacystorages.php

@@ -0,0 +1,221 @@
+<?php
+/**
+ * Copyright (c) 2013 Robin Appelman <icewind@owncloud.com>
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\Repair;
+
+use OC\Hooks\BasicEmitter;
+
+class RepairLegacyStorages extends BasicEmitter {
+	/**
+	 * @var \OCP\IConfig
+	 */
+	protected $config;
+
+	/**
+	 * @var \OC\DB\Connection
+	 */
+	protected $connection;
+
+	protected $findStorageInCacheStatement;
+	protected $renameStorageStatement;
+
+	/**
+	 * @param \OCP\IConfig $config
+	 * @param \OC\DB\Connection $connection
+	 */
+	public function __construct($config, $connection) {
+		$this->connection = $connection;
+		$this->config = $config;
+
+		$this->findStorageInCacheStatement = $this->connection->prepare(
+			'SELECT DISTINCT `storage` FROM `*PREFIX*filecache`'
+			. ' WHERE `storage` in (?, ?)'
+		);
+		$this->renameStorageStatement = $this->connection->prepare(
+			'UPDATE `*PREFIX*storages`'
+			. ' SET `id` = ?'
+			. ' WHERE `id` = ?'
+		);
+	}
+
+	public function getName() {
+		return 'Repair legacy storages';
+	}
+
+	/**
+	 * Extracts the user id	from a legacy storage id
+	 *
+	 * @param string $storageId legacy storage id in the
+	 * format "local::/path/to/datadir/userid"
+	 * @return string user id extracted from the storage id
+	 */
+	private function extractUserId($storageId) {
+		$storageId = rtrim($storageId, '/');
+		$pos = strrpos($storageId, '/');
+		return substr($storageId, $pos + 1);
+	}
+
+	/**
+	 * Fix the given legacy storage by renaming the old id
+	 * to the new id. If the new id already exists, whichever
+	 * storage that has data in the file cache will be used.
+	 * If both have data, nothing will be done and false is
+	 * returned.
+	 *
+	 * @param string $oldId old storage id
+	 * @param int $oldNumericId old storage numeric id
+	 *
+	 * @return bool true if fixed, false otherwise
+	 */
+	private function fixLegacyStorage($oldId, $oldNumericId, $userId = null) {
+		// check whether the new storage already exists
+		if (is_null($userId)) {
+			$userId = $this->extractUserId($oldId);
+		}
+		$newId = 'home::' . $userId;
+
+		// check if target id already exists
+		$newNumericId = \OC\Files\Cache\Storage::getNumericStorageId($newId);
+		if (!is_null($newNumericId)) {
+			$newNumericId = (int)$newNumericId;
+			// try and resolve the conflict
+			// check which one of "local::" or "home::" needs to be kept
+			$result = $this->findStorageInCacheStatement->execute(array($oldNumericId, $newNumericId));
+			$row1 = $this->findStorageInCacheStatement->fetch();
+			$row2 = $this->findStorageInCacheStatement->fetch();
+			$this->findStorageInCacheStatement->closeCursor();
+			if ($row2 !== false) {
+				// two results means both storages have data, not auto-fixable
+				throw new \OC\RepairException(
+					'Could not automatically fix legacy storage '
+					. '"' . $oldId . '" => "' . $newId . '"'
+					. ' because they both have data.'
+				);
+			}
+			if ($row1 === false || (int)$row1['storage'] === $oldNumericId) {
+				// old storage has data, then delete the empty new id
+				$toDelete = $newId;
+			} else if ((int)$row1['storage'] === $newNumericId) {
+				// new storage has data, then delete the empty old id
+				$toDelete = $oldId;
+			} else {
+				// unknown case, do not continue
+				return false;
+			}
+
+			// delete storage including file cache
+			\OC\Files\Cache\Storage::remove($toDelete);
+
+			// if we deleted the old id, the new id will be used
+			// automatically
+			if ($toDelete === $oldId) {
+				// nothing more to do
+				return true;
+			}
+		}
+
+		// rename old id to new id
+		$newId = \OC\Files\Cache\Storage::adjustStorageId($newId);
+		$oldId = \OC\Files\Cache\Storage::adjustStorageId($oldId);
+		$rowCount = $this->renameStorageStatement->execute(array($newId, $oldId));
+		$this->renameStorageStatement->closeCursor();
+		return ($rowCount === 1);
+	}
+
+	/**
+	 * Converts legacy home storage ids in the format
+	 * "local::/data/dir/path/userid/" to the new format "home::userid"
+	 */
+	public function run() {
+		// only run once
+		if ($this->config->getAppValue('core', 'repairlegacystoragesdone') === 'yes') {
+			return;
+		}
+
+		$dataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/');
+		$dataDir = rtrim($dataDir, '/') . '/';
+		$dataDirId = 'local::' . $dataDir;
+
+		$count = 0;
+
+		$this->connection->beginTransaction();
+
+		try {
+			// note: not doing a direct UPDATE with the REPLACE function
+			// because regexp search/extract is needed and it is not guaranteed
+			// to work on all database types
+			$sql = 'SELECT `id`, `numeric_id` FROM `*PREFIX*storages`'
+				. ' WHERE `id` LIKE ?'
+				. ' ORDER BY `id`';
+			$result = $this->connection->executeQuery($sql, array($dataDirId . '%'));
+			while ($row = $result->fetch()) {
+				$currentId = $row['id'];
+				// one entry is the datadir itself
+				if ($currentId === $dataDirId) {
+					continue;
+				}
+
+				if ($this->fixLegacyStorage($currentId, (int)$row['numeric_id'])) {
+					$count++;
+				}
+			}
+
+			// check for md5 ids, not in the format "prefix::"
+			$sql = 'SELECT COUNT(*) AS "c" FROM `*PREFIX*storages`'
+				. ' WHERE `id` NOT LIKE \'%::%\'';
+			$result = $this->connection->executeQuery($sql);
+			$row = $result->fetch();
+			// find at least one to make sure it's worth
+			// querying the user list
+			if ((int)$row['c'] > 0) {
+				$userManager = \OC_User::getManager();
+
+				// use chunks to avoid caching too many users in memory
+				$limit = 30;
+				$offset = 0;
+
+				do {
+					// query the next page of users
+					$results = $userManager->search('', $limit, $offset);
+					$storageIds = array();
+					$userIds = array();
+					foreach ($results as $uid => $userObject) {
+						$storageId = $dataDirId . $uid . '/';
+						if (strlen($storageId) <= 64) {
+							// skip short storage ids as they were handled in the previous section
+							continue;
+						}
+						$storageIds[$uid] = $storageId;
+					}
+
+					if (count($storageIds) > 0) {
+						// update the storages of these users
+						foreach ($storageIds as $uid => $storageId) {
+							$numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId);
+							if (!is_null($numericId) && $this->fixLegacyStorage($storageId, (int)$numericId)) {
+								$count++;
+							}
+						}
+					}
+					$offset += $limit;
+				} while (count($results) >= $limit);
+			}
+
+			$this->emit('\OC\Repair', 'info', array('Updated ' . $count . ' legacy home storage ids'));
+
+			$this->connection->commit();
+		}
+		catch (\OC\RepairException $e) {
+			$this->connection->rollback();
+			throw $e;
+		}
+
+		$this->config->setAppValue('core', 'repairlegacystoragesdone', 'yes');
+	}
+}

lib/private/repairexception.php

@@ -0,0 +1,16 @@
+<?php
+/**
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC;
+
+/**
+ * Exception thrown whenever a database/migration repair
+ * could not be done.
+ */
+class RepairException extends \Exception {
+}

tests/lib/db/mdb2schemamanager.php

@@ -12,6 +12,11 @@ namespace Test\DB;
 class MDB2SchemaManager extends \PHPUnit_Framework_TestCase {
 
 	public function tearDown() {
+		// do not drop the table for Oracle as it will create a bogus transaction
+		// that will break the following test suites requiring transactions
+		if (\OC::$server->getConfig()->getSystemValue('dbtype', 'sqlite') === 'oci') {
+			return;
+		}
 		\OC_DB::dropTable('table');
 	}
 

tests/lib/repair/repairlegacystorage.php

@@ -0,0 +1,282 @@
+<?php
+/**
+ * Copyright (c) 2014 Vincent Petry <pvince81@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+/**
+ * Tests for the converting of legacy storages to home storages.
+ *
+ * @see \OC\Repair\RepairLegacyStorages
+ */
+class TestRepairLegacyStorages extends PHPUnit_Framework_TestCase {
+
+	private $user;
+	private $repair;
+
+	private $dataDir;
+	private $oldDataDir;
+
+	private $legacyStorageId;
+	private $newStorageId;
+
+	public function setUp() {
+		$this->config = \OC::$server->getConfig();
+		$this->connection = \OC_DB::getConnection();
+		$this->oldDataDir = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data/');
+
+		$this->repair = new \OC\Repair\RepairLegacyStorages($this->config, $this->connection);
+	}
+
+	public function tearDown() {
+		\OC_User::deleteUser($this->user);
+
+		$sql = 'DELETE FROM `*PREFIX*storages`';
+		$this->connection->executeQuery($sql);
+		$sql = 'DELETE FROM `*PREFIX*filecache`';
+		$this->connection->executeQuery($sql);
+		\OCP\Config::setSystemValue('datadirectory', $this->oldDataDir);
+		$this->config->setAppValue('core', 'repairlegacystoragesdone', 'no');
+	}
+
+	function prepareSettings($dataDir, $userId) {
+		// hard-coded string as we want a predictable fixed length
+		// no data will be written there
+		$this->dataDir = $dataDir;
+		\OCP\Config::setSystemValue('datadirectory', $this->dataDir);
+
+		$this->user = $userId;
+		$this->legacyStorageId = 'local::' . $this->dataDir . $this->user . '/';
+		$this->newStorageId = 'home::' . $this->user;
+		\OC_User::createUser($this->user, $this->user);
+	}
+
+	/**
+	 * Create a storage entry
+	 *
+	 * @param string $storageId
+	 */
+	private function createStorage($storageId) {
+		$sql = 'INSERT INTO `*PREFIX*storages` (`id`)'
+			. ' VALUES (?)';
+
+		$storageId = \OC\Files\Cache\Storage::adjustStorageId($storageId);
+		$numRows = $this->connection->executeUpdate($sql, array($storageId));
+		$this->assertEquals(1, $numRows);
+
+		return \OC_DB::insertid('*PREFIX*storages');
+	}
+
+	/**
+	 * Returns the storage id based on the numeric id
+	 *
+	 * @param int $numericId numeric id of the storage
+	 * @return string storage id or null if not found
+	 */
+	private function getStorageId($storageId) {
+		$numericId = \OC\Files\Cache\Storage::getNumericStorageId($storageId);
+		if (!is_null($numericId)) {
+			return (int)$numericId;
+		}
+		return null;
+	}
+
+	/**
+	 * Create dummy data in the filecache for the given storage numeric id
+	 *
+	 * @param string $storageId storage id
+	 */
+	private function createData($storageId) {
+		$cache = new \OC\Files\Cache\Cache($storageId);
+		$cache->put(
+			'dummyfile.txt',
+			array('size' => 5, 'mtime' => 12, 'mimetype' => 'text/plain')
+		);
+	}
+
+	/**
+	 * Test that existing home storages are left alone when valid.
+	 * @dataProvider settingsProvider
+	 */
+	public function testNoopWithExistingHomeStorage($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$newStorageNumId = $this->createStorage($this->newStorageId);
+
+		$this->repair->run();
+
+		$this->assertNull($this->getStorageId($this->legacyStorageId));
+		$this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+	}
+
+	/**
+	 * Test that legacy storages are converted to home storages when
+	 * the latter does not exist.
+	 * @dataProvider settingsProvider
+	 */
+	public function testConvertLegacyToHomeStorage($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+
+		$this->repair->run();
+
+		$this->assertNull($this->getStorageId($this->legacyStorageId));
+		$this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId));
+	}
+
+	/**
+	 * Test that legacy storages are converted to home storages
+	 * when home storage already exists but has no data.
+	 * @dataProvider settingsProvider
+	 */
+	public function testConvertLegacyToExistingEmptyHomeStorage($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+		$newStorageNumId = $this->createStorage($this->newStorageId);
+
+		$this->createData($this->legacyStorageId);
+
+		$this->repair->run();
+
+		$this->assertNull($this->getStorageId($this->legacyStorageId));
+		$this->assertEquals($legacyStorageNumId, $this->getStorageId($this->newStorageId));
+	}
+
+	/**
+	 * Test that legacy storages are converted to home storages
+	 * when home storage already exists and the legacy storage
+	 * has no data.
+	 * @dataProvider settingsProvider
+	 */
+	public function testConvertEmptyLegacyToHomeStorage($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+		$newStorageNumId = $this->createStorage($this->newStorageId);
+
+		$this->createData($this->newStorageId);
+
+		$this->repair->run();
+
+		$this->assertNull($this->getStorageId($this->legacyStorageId));
+		$this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+	}
+
+	/**
+	 * Test that nothing is done when both conflicting legacy
+	 * and home storage have data.
+	 * @dataProvider settingsProvider
+	 */
+	public function testConflictNoop($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$legacyStorageNumId = $this->createStorage($this->legacyStorageId);
+		$newStorageNumId = $this->createStorage($this->newStorageId);
+
+		$this->createData($this->legacyStorageId);
+		$this->createData($this->newStorageId);
+
+		try {
+			$thrown = false;
+			$this->repair->run();
+		}
+		catch (\OC\RepairException $e) {
+			$thrown = true;
+		}
+
+		$this->assertTrue($thrown);
+
+		// storages left alone
+		$this->assertEquals($legacyStorageNumId, $this->getStorageId($this->legacyStorageId));
+		$this->assertEquals($newStorageNumId, $this->getStorageId($this->newStorageId));
+
+		// did not set the done flag
+		$this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+	}
+
+	/**
+	 * Test that the data dir local entry is left alone
+	 * @dataProvider settingsProvider
+	 */
+	public function testDataDirEntryNoop($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$storageId = 'local::' . $this->dataDir;
+		$numId = $this->createStorage($storageId);
+
+		$this->repair->run();
+
+		$this->assertEquals($numId, $this->getStorageId($storageId));
+	}
+
+	/**
+	 * Test that external local storages are left alone
+	 * @dataProvider settingsProvider
+	 */
+	public function testLocalExtStorageNoop($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$storageId = 'local::/tmp/somedir/' . $this->user;
+		$numId = $this->createStorage($storageId);
+
+		$this->repair->run();
+
+		$this->assertEquals($numId, $this->getStorageId($storageId));
+	}
+
+	/**
+	 * Test that other external storages are left alone
+	 * @dataProvider settingsProvider
+	 */
+	public function testExtStorageNoop($dataDir, $userId) {
+		$this->prepareSettings($dataDir, $userId);
+		$storageId = 'smb::user@password/tmp/somedir/' . $this->user;
+		$numId = $this->createStorage($storageId);
+
+		$this->repair->run();
+
+		$this->assertEquals($numId, $this->getStorageId($storageId));
+	}
+
+	/**
+	 * Provides data dir and user name
+	 */
+	function settingsProvider() {
+		return array(
+			// regular data dir
+			array(
+				'/tmp/oc-autotest/datadir/',
+				uniqid('user_'),
+			),
+			// long datadir / short user
+			array(
+				'/tmp/oc-autotest/datadir01234567890123456789012345678901234567890123456789END/',
+				uniqid('user_'),
+			),
+			// short datadir / long user
+			array(
+				'/tmp/oc-autotest/datadir/',
+				'u123456789012345678901234567890123456789012345678901234567890END', // 64 chars
+			),
+		);
+	}
+
+	/**
+	 * Only run the repair once
+	 */
+	public function testOnlyRunOnce() {
+		$output = array();
+		$this->repair->listen('\OC\Repair', 'info', function ($description) use (&$output) {
+			$output[] = 'info: ' . $description;
+		});
+
+		$this->prepareSettings('/tmp/oc-autotest/datadir', uniqid('user_'));
+		$this->assertNotEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+		$this->repair->run();
+		$this->assertEquals(1, count($output));
+		$this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+
+		$output = array();
+		$this->repair->run();
+		// no output which means it did not run
+		$this->assertEquals(0, count($output));
+		$this->assertEquals('yes', $this->config->getAppValue('core', 'repairlegacystoragesdone'));
+	}
+}