From 0a41b906ca03d65799babaf1c55e248ad6158598 Mon Sep 17 00:00:00 2001 From: Yannick Warnier Date: Tue, 6 Nov 2012 04:23:38 -0500 Subject: [PATCH] Improved migration scripts for a particular case - refs #4881 --- tests/migrate/config.dist.php | 24 +++------ tests/migrate/migrate.php | 65 +++++++++++++----------- tests/migrate/migration.class.php | 36 ++++++++----- tests/migrate/migration.custom.class.php | 51 +++++++++++++++---- tests/migrate/migration.mssql.class.php | 6 +-- 5 files changed, 107 insertions(+), 75 deletions(-) diff --git a/tests/migrate/config.dist.php b/tests/migrate/config.dist.php index 7fcb16d12a..b0a670c298 100644 --- a/tests/migrate/config.dist.php +++ b/tests/migrate/config.dist.php @@ -18,7 +18,7 @@ $db_type = 'mssql'; * database is located. This name should be routeable by PHP. * Defaults to: localhost */ -$db_host = 'localhost'; +//$db_host = 'localhost'; /* * The database port is the port on which to connect on the origin * database host. The default port for MS-SQL is 1433, which we @@ -29,26 +29,19 @@ $db_port = '1433'; * The database user is the name under which to connect to the * origin database server. Defaults to: lms */ -$db_user = 'lms'; +$db_user = 'user_of_db'; /* * The database password is the password for the user on the origin * database server. Defaults to: password */ -$db_pass = 'password'; +$db_pass = '**********'; /* * The database name on the database origin server. * Defaults to: master */ -$db_name = 'master'; +$db_name = 'master1'; //second DB -$db_name2 = 'master2'; -/** - * Boost the migration by putting the relations tables in memory (as well as - * in the database). This might use huge amounts of memory when managing - * users bases of several hundred thousands, so the default is to disable it - */ -$boost = array('users'=>false, 'courses'=>false, 'sessions'=>false); - +$db_name2 = 'master'; $config = array( @@ -59,7 +52,6 @@ $config = array( 'db_pass' => $db_pass, 'db_name' => $db_name, ); - $config2 = array( 'type' => $db_type, 'host' => $db_host, @@ -73,14 +65,12 @@ $servers = array( array( 'name' => 'Old ms', 'filename' => 'db_matches.php', 'connection' => $config, - 'active' => false, - 'action' => 'm', //m for migrate, s for synchronize + 'active' => false ), array( 'name' => 'with e class stuff', 'filename' => 'db_matches_2.php', 'connection' => $config2, - 'active' => true, - 'action' => 'm', //m for migrate, s for synchronize + 'active' => true ), ); diff --git a/tests/migrate/migrate.php b/tests/migrate/migrate.php index b6eaf2de44..1e668a34a6 100644 --- a/tests/migrate/migrate.php +++ b/tests/migrate/migrate.php @@ -1,9 +1,5 @@ connect(); - if ($server_info['m']) { - /** - * Prepare the arrays of matches that will allow for the migration - */ - $migrate = array(); - include $server_info['filename']; - //Default migration from MSSQL to Chamilo MySQL - $m->migrate($matches); - } elseif ($server_info['s']) { - //Getting transactions from MSSQL (via webservices) - if (isset($matches['web_service_calls']['filename'])) { - require_once $matches['web_service_calls']['filename']; - } - //This functions truncates the transaction lists! - $m->test_transactions($matches['web_service_calls']); - //$m->search_transactions($matches['web_service_calls']); - //Load transactions saved before - $m->load_transactions($matches); + /** + * Prepare the arrays of matches that will allow for the migration + */ + $migrate = array(); + include $server_info['filename']; + //Default migration from MSSQL to Chamilo MySQL + $m->migrate($matches); + + //Getting transactions from MSSQL (via webservices) + + if (isset($matches['web_service_calls']['filename'])) { + require_once $matches['web_service_calls']['filename']; } + //This functions truncates the transaction lists! + //$m->test_transactions($matches['web_service_calls']); + + //$m->search_transactions($matches['web_service_calls']); + + //Load transactions saved before + //$m->load_transactions($matches); + + //print_r($m->errors_stack); + //echo "OK so far\n"; echo "\n ---- End loading server----- \n"; } else { error_log("db_matches not activated: {$server_info['name']} {$server_info['filename']}"); - } + } } } +$end = time(); +echo "Total process took ".($end-$start)." seconds"; diff --git a/tests/migrate/migration.class.php b/tests/migrate/migration.class.php index ae11b105dd..ddac0575bb 100644 --- a/tests/migrate/migration.class.php +++ b/tests/migrate/migration.class.php @@ -55,15 +55,14 @@ class Migration { * data. Store values here (preferably using the same indexes as the * destination database field names) until ready to insert into Chamilo. */ - public $data_list = array('users' => array(), 'courses' => array(), 'sessions' => array()); - /** - * Whether to use the memory instead of the database (if the memory allows) - * This option has to be configured manually, and the default is to use - * the database, which is slower but much safer in terms of stability of - * the script (or let's say much more linear in terms of execution time) - */ - private $boost = array('users' => false, 'courses' => false, 'sessions' => false); - + public $data_list = array( + 'boost_users' => false, + 'users' => array(), + 'boost_courses' => false, + 'courses' => array(), + 'boost_sessions' => false, + 'sessions' => array(), + ); /** * The constructor assigns all database connection details to the migration * object @@ -72,7 +71,6 @@ class Migration { * @param string The original database's user * @param string The original database's password * @param string The original database's name - * @param array Array of parameters to define the memory-boost behaviour * @return boolean False on error. Void on success. */ public function __construct($dbhost, $dbport, $dbuser, $dbpass, $dbname, $boost=false) { @@ -88,7 +86,11 @@ class Migration { $this->odbname = $dbname; // Set the boost level if set in config.php if (!empty($boost) && is_array($boost)) { - $this->boost = $boost; + foreach ($boost as $item => $val) { + if ($val == true) { + $this->data_list[$item] = true; + } + } } } @@ -105,11 +107,12 @@ class Migration { * @param array Structured array of matches (see migrate.php) */ public function migrate($matches) { - error_log("\n" . '------------ Migration->migrate function called ------------' . "\n"); + error_log("\n" . '------------ ['.date('H:i:s').'] Migration->migrate function called ------------' . "\n"); $extra_fields = array(); // Browsing through 1st-level arrays in db_matches.php foreach ($matches as $table) { - error_log('Found table ' . $table['orig_table'] . ' in db_matches'); + echo "Starting table ".$table['orig_table']." at ".date('h:i:s')."\n"; + error_log('['.date('H:i:s').'] Found table ' . $table['orig_table'] . ' in db_matches'); $build_only = false; if (empty($table['dest_table'])) { @@ -779,6 +782,7 @@ class Migration { if ($details['func'] == 'none' || empty($details['func'])) { $dest_data = $row[$details['orig']]; } else { + //error_log(__FILE__.' '.__LINE__.' Preparing to treat field with '.$details['func']); $dest_data = MigrationCustom::$details['func']($row[$details['orig']], $this->data_list, $row); } @@ -798,8 +802,10 @@ class Migration { $field_type = $extra_field['field_type']; if (!empty($options)) { + if (!is_array($options)) { $options = array($options); } foreach ($options as $option) { - foreach ($option as $key => $value) { + if (is_array($option)) { + foreach ($option as $key => $value) { //error_log("$key $value --> {$dest_row[$details['dest']]} "); if ($key == 'option_value' && $value == $dest_row[$details['dest']]) { $value = $option['option_display_text']; @@ -813,6 +819,7 @@ class Migration { break(2); } } + } } } } else { @@ -866,6 +873,7 @@ class Migration { } } else { global $api_failureList; + error_log('Empty user details'); error_log(print_r($api_failureList, 1)); } break; diff --git a/tests/migrate/migration.custom.class.php b/tests/migrate/migration.custom.class.php index 708f4949aa..d3c35bd64a 100644 --- a/tests/migrate/migration.custom.class.php +++ b/tests/migrate/migration.custom.class.php @@ -64,11 +64,16 @@ class MigrationCustom { return utf8_encode($value); } - static function clean_session_name($value, &$omigrate, $row_data) { + static function clean_session_name($value, $omigrate, $row_data) { return self::clean_utf8($row_data['session_name']); } - static function get_real_course_code($data) { + static function get_real_course_code($data, $omigrate=null) { + if (is_object($omigrate) && $omigrate->boost_courses) { + if (isset($omigrate->courses[$data])) { + return $omigrate->courses[$data]; + } + } $extra_field = new ExtraFieldValue('course'); $values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdCurso', $data); if ($values) { @@ -78,7 +83,12 @@ class MigrationCustom { } } - static function get_session_id_by_programa_id($uidIdProgram) { + static function get_session_id_by_programa_id($uidIdProgram, $omigrate=null) { + if (is_object($omigrate) && $omigrate->boost_sessions) { + if (isset($omigrate->sessions[$uidIdPrograma])) { + return $omigrate->sessions[$uidIdPrograma]; + } + } $extra_field = new ExtraFieldValue('session'); $values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPrograma', $uidIdProgram); if ($values) { @@ -89,7 +99,12 @@ class MigrationCustom { } /* Not used */ - static function get_user_id_by_persona_id($uidIdPersona) { + static function get_user_id_by_persona_id($uidIdPersona, $omigrate=null) { + if (is_object($omigrate) && $omigrate->boost_users) { + if (isset($omigrate->users[$uidIdPersona])) { + return $omigrate->users[$uidIdPersona]; + } + } //error_log('get_user_id_by_persona_id'); $extra_field = new ExtraFieldValue('user'); $values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPersona', $uidIdPersona); @@ -100,12 +115,17 @@ class MigrationCustom { } } - static function get_real_teacher_id($uidIdPersona) { + static function get_real_teacher_id($uidIdPersona, $omigrate=null) { $default_teacher_id = self::default_admin_id; if (empty($uidIdPersona)) { //error_log('No teacher provided'); return $default_teacher_id; } + if (is_object($omigrate) && $omigrate->boost_users) { + if (isset($omigrate->users[$uidIdPersona])) { + return $omigrate->users[$uidIdPersona]; + } + } $extra_field = new ExtraFieldValue('user'); $values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPersona', $uidIdPersona); @@ -244,7 +264,12 @@ class MigrationCustom { //error_log(print_r($data, 1)); $user_info = UserManager::add($data); if (!$user_info) { - echo 'error'; + error_log('User '.$id_persona.' could not be inserted (maybe duplicate?)'); + } else { + //error_log('User '.$id_persona.' was created as user '.$user_info['user_id']); + } + if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_users) { + $omigrate->users[$id_persona] = $user_info['user_id']; } UserManager::update_extra_field_value($user_info['user_id'], 'uidIdPersona', $id_persona); return $user_info; @@ -253,7 +278,7 @@ class MigrationCustom { /** * Manages the course creation based on the rules in db_matches.php */ - static function create_course($data) { + static function create_course($data, $omigrate) { //error_log('In create_course, received '.print_r($data,1)); //Fixes wrong wanted codes $data['wanted_code'] = str_replace(array('-', '_'), '000', $data['wanted_code']); @@ -272,6 +297,9 @@ class MigrationCustom { 'max' => '20' );*/ $course_data = CourseManager::create_course($data); + if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_courses) { + $omigrate->courses[$data['uidIdCurso']] = $course_data['code']; + } return $course_data; } @@ -279,7 +307,7 @@ class MigrationCustom { * Manages the session creation, based on data provided by the rules * in db_matches.php */ - static function create_session($data) { + static function create_session($data, $omigrate) { //Hack to add the default gradebook course to the session course $data['create_gradebook_evaluation'] = true; /*$data['gradebook_params'] = array( @@ -298,7 +326,10 @@ class MigrationCustom { //print_r($data); //exit; } else{ - //error_log('session_id created'); + //error_log('session_id created'); + if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_sessions) { + $omigrate->sessions[$data['uidIdPrograma']] = $session_id; + } } return $session_id; } @@ -1188,4 +1219,4 @@ class MigrationCustom { static function transaction_meses_editar($data, $web_service_details) { self::transaction_extra_field_editar_generic('meses', $data, $web_service_details); } -} \ No newline at end of file +} diff --git a/tests/migrate/migration.mssql.class.php b/tests/migrate/migration.mssql.class.php index fbe8af7010..924e1433e1 100644 --- a/tests/migrate/migration.mssql.class.php +++ b/tests/migrate/migration.mssql.class.php @@ -33,13 +33,13 @@ class MigrationMSSQL extends Migration { //In order to process X item of each table add TOP X $top = null; - //$top = " TOP 1000 "; +// $top = " TOP 100000 "; if (in_array($table, array('Empleado', 'Alumno'))) { - $top = " TOP 10000 "; + // $top = " TOP 10000 "; } if (in_array($table, array('ProgramaAcademico', 'Matricula'))) { - $top = " TOP 100000 "; + // $top = " TOP 100000 "; } //$top = null;