Merge pull request #7015 from owncloud/db-migration-check-copy

Check database migration on a copy of the table first

lib/private/db/mdb2schemamanager.php

@@ -8,6 +8,11 @@
 
 namespace OC\DB;
 
+use Doctrine\DBAL\Platforms\MySqlPlatform;
+use Doctrine\DBAL\Platforms\OraclePlatform;
+use Doctrine\DBAL\Platforms\PostgreSqlPlatform;
+use Doctrine\DBAL\Platforms\SqlitePlatform;
+
 class MDB2SchemaManager {
 	/**
 	 * @var \OC\DB\Connection $conn
@@ -31,7 +36,7 @@ class MDB2SchemaManager {
 	 *
 	 * TODO: write more documentation
 	 */
-	public function getDbStructure( $file, $mode = MDB2_SCHEMA_DUMP_STRUCTURE) {
+	public function getDbStructure($file, $mode = MDB2_SCHEMA_DUMP_STRUCTURE) {
 		$sm = $this->conn->getSchemaManager();
 
 		return \OC_DB_MDB2SchemaWriter::saveSchemaToFile($file, $sm);
@@ -44,58 +49,64 @@ class MDB2SchemaManager {
 	 *
 	 * TODO: write more documentation
 	 */
-	public function createDbFromStructure( $file ) {
+	public function createDbFromStructure($file) {
 		$schemaReader = new MDB2SchemaReader(\OC_Config::getObject(), $this->conn->getDatabasePlatform());
 		$toSchema = $schemaReader->loadSchemaFromFile($file);
 		return $this->executeSchemaChange($toSchema);
 	}
 
+	/**
+	 * @return \OC\DB\Migrator
+	 */
+	protected function getMigrator() {
+		$platform = $this->conn->getDatabasePlatform();
+		if ($platform instanceof SqlitePlatform) {
+			return new SQLiteMigrator($this->conn);
+		} else if ($platform instanceof OraclePlatform) {
+			return new OracleMigrator($this->conn);
+		} else if ($platform instanceof MySqlPlatform) {
+			return new MySQLMigrator($this->conn);
+		} else if ($platform instanceof PostgreSqlPlatform) {
+			return new Migrator($this->conn);
+		} else {
+			return new NoCheckMigrator($this->conn);
+		}
+	}
+
 	/**
 	 * update the database scheme
 	 * @param string $file file to read structure from
+	 * @param bool $generateSql only return the sql needed for the upgrade
 	 * @return string|boolean
 	 */
 	public function updateDbFromStructure($file, $generateSql = false) {
-		$sm = $this->conn->getSchemaManager();
-		$fromSchema = $sm->createSchema();
-
-		$schemaReader = new MDB2SchemaReader(\OC_Config::getObject(), $this->conn->getDatabasePlatform());
-		$toSchema = $schemaReader->loadSchemaFromFile($file);
-
-		// remove tables we don't know about
-		/** @var $table \Doctrine\DBAL\Schema\Table */
-		foreach($fromSchema->getTables() as $table) {
-			if (!$toSchema->hasTable($table->getName())) {
-				$fromSchema->dropTable($table->getName());
-			}
-		}
-		// remove sequences we don't know about
-		foreach($fromSchema->getSequences() as $table) {
-			if (!$toSchema->hasSequence($table->getName())) {
-				$fromSchema->dropSequence($table->getName());
-			}
-		}
-
-		$comparator = new \Doctrine\DBAL\Schema\Comparator();
-		$schemaDiff = $comparator->compare($fromSchema, $toSchema);
 
 		$platform = $this->conn->getDatabasePlatform();
-		foreach($schemaDiff->changedTables as $tableDiff) {
-			$tableDiff->name = $platform->quoteIdentifier($tableDiff->name);
-			foreach($tableDiff->changedColumns as $column) {
-				$column->oldColumnName = $platform->quoteIdentifier($column->oldColumnName);
-			}
-		}
+		$schemaReader = new MDB2SchemaReader(\OC_Config::getObject(), $platform);
+		$toSchema = $schemaReader->loadSchemaFromFile($file);
+		$migrator = $this->getMigrator();
 
 		if ($generateSql) {
-			return $this->generateChangeScript($schemaDiff);
+			return $migrator->generateChangeScript($toSchema);
+		} else {
+			$migrator->checkMigrate($toSchema);
+			$migrator->migrate($toSchema);
+			return true;
 		}
+	}
 
-		return $this->executeSchemaChange($schemaDiff);
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $schema
+	 * @return string
+	 */
+	public function generateChangeScript($schema) {
+		$migrator = $this->getMigrator();
+		return $migrator->generateChangeScript($schema);
 	}
 
 	/**
 	 * remove all tables defined in a database structure xml file
+	 *
 	 * @param string $file the xml file describing the tables
 	 */
 	public function removeDBStructure($file) {
@@ -103,7 +114,7 @@ class MDB2SchemaManager {
 		$fromSchema = $schemaReader->loadSchemaFromFile($file);
 		$toSchema = clone $fromSchema;
 		/** @var $table \Doctrine\DBAL\Schema\Table */
-		foreach($toSchema->getTables() as $table) {
+		foreach ($toSchema->getTables() as $table) {
 			$toSchema->dropTable($table->getName());
 		}
 		$comparator = new \Doctrine\DBAL\Schema\Comparator();
@@ -117,26 +128,10 @@ class MDB2SchemaManager {
 	 */
 	private function executeSchemaChange($schema) {
 		$this->conn->beginTransaction();
-		foreach($schema->toSql($this->conn->getDatabasePlatform()) as $sql) {
+		foreach ($schema->toSql($this->conn->getDatabasePlatform()) as $sql) {
 			$this->conn->query($sql);
 		}
 		$this->conn->commit();
 		return true;
 	}
-
-	/**
-	 * @param \Doctrine\DBAL\Schema\Schema $schema
-	 * @return string
-	 */
-	public function generateChangeScript($schema) {
-
-		$script = '';
-		$sqls = $schema->toSql($this->conn->getDatabasePlatform());
-		foreach($sqls as $sql) {
-			$script .= $sql . ';';
-			$script .= PHP_EOL;
-		}
-
-		return $script;
-	}
 }

lib/private/db/migrationexception.php

@@ -0,0 +1,26 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+
+class MigrationException extends \Exception {
+	private $table;
+
+	public function __construct($table, $message) {
+		$this->table = $table;
+		parent::__construct($message);
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getTable() {
+		return $this->table;
+	}
+}

lib/private/db/migrator.php

@@ -0,0 +1,196 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+use \Doctrine\DBAL\DBALException;
+use \Doctrine\DBAL\Schema\Index;
+use \Doctrine\DBAL\Schema\Table;
+use \Doctrine\DBAL\Schema\Schema;
+use \Doctrine\DBAL\Schema\SchemaConfig;
+use \Doctrine\DBAL\Schema\Comparator;
+
+class Migrator {
+	/**
+	 * @var \Doctrine\DBAL\Connection $connection
+	 */
+	protected $connection;
+
+	/**
+	 * @param \Doctrine\DBAL\Connection $connection
+	 */
+	public function __construct(\Doctrine\DBAL\Connection $connection) {
+		$this->connection = $connection;
+	}
+
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $targetSchema
+	 */
+	public function migrate(Schema $targetSchema) {
+		$this->applySchema($targetSchema);
+	}
+
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $targetSchema
+	 * @return string
+	 */
+	public function generateChangeScript(Schema $targetSchema) {
+		$schemaDiff = $this->getDiff($targetSchema, $this->connection);
+
+		$script = '';
+		$sqls = $schemaDiff->toSql($this->connection->getDatabasePlatform());
+		foreach ($sqls as $sql) {
+			$script .= $sql . ';';
+			$script .= PHP_EOL;
+		}
+
+		return $script;
+	}
+
+	/**
+	 * @param Schema $targetSchema
+	 * @throws \OC\DB\MigrationException
+	 */
+	public function checkMigrate(Schema $targetSchema) {
+		/**
+		 * @var \Doctrine\DBAL\Schema\Table[] $tables
+		 */
+		$tables = $targetSchema->getTables();
+
+		$existingTables = $this->connection->getSchemaManager()->listTableNames();
+
+		foreach ($tables as $table) {
+			if (strpos($table->getName(), '.')) {
+				list(, $tableName) = explode('.', $table->getName());
+			} else {
+				$tableName = $table->getName();
+			}
+			// don't need to check for new tables
+			if (array_search($tableName, $existingTables) !== false) {
+				$this->checkTableMigrate($table);
+			}
+		}
+	}
+
+	/**
+	 * Create a unique name for the temporary table
+	 *
+	 * @param string $name
+	 * @return string
+	 */
+	protected function generateTemporaryTableName($name) {
+		return 'oc_' . $name . '_' . uniqid();
+	}
+
+	/**
+	 * Check the migration of a table on a copy so we can detect errors before messing with the real table
+	 *
+	 * @param \Doctrine\DBAL\Schema\Table $table
+	 * @throws \OC\DB\MigrationException
+	 */
+	protected function checkTableMigrate(Table $table) {
+		$name = $table->getName();
+		$tmpName = $this->generateTemporaryTableName($name);
+
+		$this->copyTable($name, $tmpName);
+
+		//create the migration schema for the temporary table
+		$tmpTable = $this->renameTableSchema($table, $tmpName);
+		$schemaConfig = new SchemaConfig();
+		$schemaConfig->setName($this->connection->getDatabase());
+		$schema = new Schema(array($tmpTable), array(), $schemaConfig);
+
+		try {
+			$this->applySchema($schema);
+			$this->dropTable($tmpName);
+		} catch (DBALException $e) {
+			// pgsql needs to commit it's failed transaction before doing anything else
+			$this->connection->commit();
+			$this->dropTable($tmpName);
+			throw new MigrationException($table->getName(), $e->getMessage());
+		}
+	}
+
+	/**
+	 * @param \Doctrine\DBAL\Schema\Table $table
+	 * @param string $newName
+	 * @return \Doctrine\DBAL\Schema\Table
+	 */
+	protected function renameTableSchema(Table $table, $newName) {
+		/**
+		 * @var \Doctrine\DBAL\Schema\Index[] $indexes
+		 */
+		$indexes = $table->getIndexes();
+		$newIndexes = array();
+		foreach ($indexes as $index) {
+			$indexName = 'oc_' . uniqid(); // avoid conflicts in index names
+			$newIndexes[] = new Index($indexName, $index->getColumns(), $index->isUnique(), $index->isPrimary());
+		}
+
+		// foreign keys are not supported so we just set it to an empty array
+		return new Table($newName, $table->getColumns(), $newIndexes, array(), 0, $table->getOptions());
+	}
+
+	protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) {
+		$sourceSchema = $connection->getSchemaManager()->createSchema();
+
+		// remove tables we don't know about
+		/** @var $table \Doctrine\DBAL\Schema\Table */
+		foreach ($sourceSchema->getTables() as $table) {
+			if (!$targetSchema->hasTable($table->getName())) {
+				$sourceSchema->dropTable($table->getName());
+			}
+		}
+		// remove sequences we don't know about
+		foreach ($sourceSchema->getSequences() as $table) {
+			if (!$targetSchema->hasSequence($table->getName())) {
+				$sourceSchema->dropSequence($table->getName());
+			}
+		}
+
+		$comparator = new Comparator();
+		return $comparator->compare($sourceSchema, $targetSchema);
+	}
+
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $targetSchema
+	 * @param \Doctrine\DBAL\Connection $connection
+	 */
+	protected function applySchema(Schema $targetSchema, \Doctrine\DBAL\Connection $connection = null) {
+		if (is_null($connection)) {
+			$connection = $this->connection;
+		}
+
+		$schemaDiff = $this->getDiff($targetSchema, $connection);
+
+		$connection->beginTransaction();
+		foreach ($schemaDiff->toSql($connection->getDatabasePlatform()) as $sql) {
+			$connection->query($sql);
+		}
+		$connection->commit();
+	}
+
+	/**
+	 * @param string $sourceName
+	 * @param string $targetName
+	 */
+	protected function copyTable($sourceName, $targetName) {
+		$quotedSource = $this->connection->quoteIdentifier($sourceName);
+		$quotedTarget = $this->connection->quoteIdentifier($targetName);
+
+		$this->connection->exec('CREATE TABLE ' . $quotedTarget . ' (LIKE ' . $quotedSource . ')');
+		$this->connection->exec('INSERT INTO ' . $quotedTarget . ' SELECT * FROM ' . $quotedSource);
+	}
+
+	/**
+	 * @param string $name
+	 */
+	protected function dropTable($name) {
+		$this->connection->exec('DROP TABLE ' . $this->connection->quoteIdentifier($name));
+	}
+}

lib/private/db/mysqlmigrator.php

@@ -0,0 +1,32 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+use Doctrine\DBAL\Schema\Schema;
+
+class MySQLMigrator extends Migrator {
+	/**
+	 * @param Schema $targetSchema
+	 * @param \Doctrine\DBAL\Connection $connection
+	 * @return \Doctrine\DBAL\Schema\SchemaDiff
+	 */
+	protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) {
+		$schemaDiff = parent::getDiff($targetSchema, $connection);
+
+		// identifiers need to be quoted for mysql
+		foreach ($schemaDiff->changedTables as $tableDiff) {
+			$tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name);
+			foreach ($tableDiff->changedColumns as $column) {
+				$column->oldColumnName = $this->connection->quoteIdentifier($column->oldColumnName);
+			}
+		}
+
+		return $schemaDiff;
+	}
+}

lib/private/db/nocheckmigrator.php

@@ -0,0 +1,24 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+use Doctrine\DBAL\Schema\Schema;
+
+/**
+ * migrator for database platforms that don't support the upgrade check
+ *
+ * @package OC\DB
+ */
+class NoCheckMigrator extends Migrator {
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $targetSchema
+	 * @throws \OC\DB\MigrationException
+	 */
+	public function checkMigrate(Schema $targetSchema) {}
+}

lib/private/db/oraclemigrator.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+use Doctrine\DBAL\Schema\Schema;
+
+class OracleMigrator extends NoCheckMigrator {
+	/**
+	 * @param Schema $targetSchema
+	 * @param \Doctrine\DBAL\Connection $connection
+	 * @return \Doctrine\DBAL\Schema\SchemaDiff
+	 */
+	protected function getDiff(Schema $targetSchema, \Doctrine\DBAL\Connection $connection) {
+		$schemaDiff = parent::getDiff($targetSchema, $connection);
+
+		// oracle forces us to quote the identifiers
+		foreach ($schemaDiff->changedTables as $tableDiff) {
+			$tableDiff->name = $this->connection->quoteIdentifier($tableDiff->name);
+			foreach ($tableDiff->changedColumns as $column) {
+				$column->oldColumnName = $this->connection->quoteIdentifier($column->oldColumnName);
+			}
+		}
+
+		return $schemaDiff;
+	}
+
+	/**
+	 * @param string $name
+	 * @return string
+	 */
+	protected function generateTemporaryTableName($name) {
+		return 'oc_' . uniqid();
+	}
+}

lib/private/db/sqlitemigrator.php

@@ -0,0 +1,40 @@
+<?php
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\DB;
+
+use Doctrine\DBAL\DBALException;
+
+class SQLiteMigrator extends Migrator {
+	/**
+	 * @param \Doctrine\DBAL\Schema\Schema $targetSchema
+	 * @throws \OC\DB\MigrationException
+	 *
+	 * For sqlite we simple make a copy of the entire database, and test the migration on that
+	 */
+	public function checkMigrate(\Doctrine\DBAL\Schema\Schema $targetSchema) {
+		$dbFile = $this->connection->getDatabase();
+		$tmpFile = \OC_Helper::tmpFile('.db');
+		copy($dbFile, $tmpFile);
+
+		$connectionParams = array(
+			'path' => $tmpFile,
+			'driver' => 'pdo_sqlite',
+		);
+		$conn = \Doctrine\DBAL\DriverManager::getConnection($connectionParams);
+		try {
+			$this->applySchema($targetSchema, $conn);
+			$conn->close();
+			unlink($tmpFile);
+		} catch (DBALException $e) {
+			$conn->close();
+			unlink($tmpFile);
+			throw new MigrationException('', $e->getMessage());
+		}
+	}
+}

tests/lib/db/migrator.php

@@ -0,0 +1,122 @@
+<?php
+
+/**
+ * Copyright (c) 2014 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace Test\DB;
+
+use \Doctrine\DBAL\DBALException;
+use \Doctrine\DBAL\Schema\Schema;
+use \Doctrine\DBAL\Schema\SchemaConfig;
+
+class Migrator extends \PHPUnit_Framework_TestCase {
+	/**
+	 * @var \Doctrine\DBAL\Connection $connection
+	 */
+	private $connection;
+
+	private $tableName;
+
+	public function setUp() {
+		$this->connection = \OC_DB::getConnection();
+		if ($this->connection->getDriver() instanceof \Doctrine\DBAL\Driver\OCI8\Driver) {
+			$this->markTestSkipped('DB migration tests arent supported on OCI');
+		}
+		$this->tableName = 'test_' . uniqid();
+	}
+
+	public function tearDown() {
+		$this->connection->exec('DROP TABLE ' . $this->tableName);
+	}
+
+	/**
+	 * @return \Doctrine\DBAL\Schema\Schema[]
+	 */
+	private function getDuplicateKeySchemas() {
+		$startSchema = new Schema(array(), array(), $this->getSchemaConfig());
+		$table = $startSchema->createTable($this->tableName);
+		$table->addColumn('id', 'integer');
+		$table->addColumn('name', 'string');
+		$table->addIndex(array('id'), $this->tableName . '_id');
+
+		$endSchema = new Schema(array(), array(), $this->getSchemaConfig());
+		$table = $endSchema->createTable($this->tableName);
+		$table->addColumn('id', 'integer');
+		$table->addColumn('name', 'string');
+		$table->addUniqueIndex(array('id'), $this->tableName . '_id');
+
+		return array($startSchema, $endSchema);
+	}
+
+	private function getSchemaConfig() {
+		$config = new SchemaConfig();
+		$config->setName($this->connection->getDatabase());
+		return $config;
+	}
+
+	private function isSQLite() {
+		return $this->connection->getDriver() instanceof \Doctrine\DBAL\Driver\PDOSqlite\Driver;
+	}
+
+	private function getMigrator() {
+		if ($this->isSQLite()) {
+			return new \OC\DB\SQLiteMigrator($this->connection);
+		} else {
+			return new \OC\DB\Migrator($this->connection);
+		}
+	}
+
+	/**
+	 * @expectedException \OC\DB\MigrationException
+	 */
+	public function testDuplicateKeyUpgrade() {
+		if ($this->isSQLite()) {
+			$this->markTestSkipped('sqlite doesnt throw errors when creating a new key on existing data');
+		}
+		list($startSchema, $endSchema) = $this->getDuplicateKeySchemas();
+		$migrator = $this->getMigrator();
+		$migrator->migrate($startSchema);
+
+		$this->connection->insert($this->tableName, array('id' => 1, 'name' => 'foo'));
+		$this->connection->insert($this->tableName, array('id' => 2, 'name' => 'bar'));
+		$this->connection->insert($this->tableName, array('id' => 2, 'name' => 'qwerty'));
+
+		$migrator->checkMigrate($endSchema);
+		$this->fail('checkMigrate should have failed');
+	}
+
+	public function testUpgrade() {
+		list($startSchema, $endSchema) = $this->getDuplicateKeySchemas();
+		$migrator = $this->getMigrator();
+		$migrator->migrate($startSchema);
+
+		$this->connection->insert($this->tableName, array('id' => 1, 'name' => 'foo'));
+		$this->connection->insert($this->tableName, array('id' => 2, 'name' => 'bar'));
+		$this->connection->insert($this->tableName, array('id' => 3, 'name' => 'qwerty'));
+
+		$migrator->checkMigrate($endSchema);
+		$migrator->migrate($endSchema);
+		$this->assertTrue(true);
+	}
+
+	public function testInsertAfterUpgrade() {
+		list($startSchema, $endSchema) = $this->getDuplicateKeySchemas();
+		$migrator = $this->getMigrator();
+		$migrator->migrate($startSchema);
+
+		$migrator->migrate($endSchema);
+
+		$this->connection->insert($this->tableName, array('id' => 1, 'name' => 'foo'));
+		$this->connection->insert($this->tableName, array('id' => 2, 'name' => 'bar'));
+		try {
+			$this->connection->insert($this->tableName, array('id' => 2, 'name' => 'qwerty'));
+			$this->fail('Expected duplicate key insert to fail');
+		} catch (DBALException $e) {
+			$this->assertTrue(true);
+		}
+	}
+}