chamilo
/
chamilo-lms
mirror of https://github.com/chamilo/chamilo-lms/tree/1.11.x
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Improved migration scripts for a particular case - refs #4881

Browse Source
skala
Yannick Warnier 13 years ago
parent 7feb5f6280
commit 0a41b906ca
5 changed files with 107 additions and 75 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 24
      tests/migrate/config.dist.php
  2. 65
      tests/migrate/migrate.php
  3. 36
      tests/migrate/migration.class.php
  4. 51
      tests/migrate/migration.custom.class.php
  5. 6
      tests/migrate/migration.mssql.class.php

24
tests/migrate/config.dist.php
Unescape View File

@ -18,7 +18,7 @@ $db_type = 'mssql';
* database is located. This name should be routeable by PHP.
* Defaults to: localhost
*/
$db_host = 'localhost';
//$db_host = 'localhost';
/*
* The database port is the port on which to connect on the origin
* database host. The default port for MS-SQL is 1433, which we
@ -29,26 +29,19 @@ $db_port = '1433';
* The database user is the name under which to connect to the
* origin database server. Defaults to: lms
*/
$db_user = 'lms';
$db_user = 'user_of_db';
/*
* The database password is the password for the user on the origin
* database server. Defaults to: password
*/
$db_pass = 'password';
$db_pass = '**********';
/*
* The database name on the database origin server.
* Defaults to: master
*/
$db_name = 'master';
$db_name = 'master1';
//second DB
$db_name2 = 'master2';
/**
* Boost the migration by putting the relations tables in memory (as well as
* in the database). This might use huge amounts of memory when managing
* users bases of several hundred thousands, so the default is to disable it
*/
$boost = array('users'=>false, 'courses'=>false, 'sessions'=>false);
$db_name2 = 'master';
$config = array(
@ -59,7 +52,6 @@ $config = array(
'db_pass' => $db_pass,
'db_name' => $db_name,
);
$config2 = array(
'type' => $db_type,
'host' => $db_host,
@ -73,14 +65,12 @@ $servers = array(
array( 'name' => 'Old ms',
'filename' => 'db_matches.php',
'connection' => $config,
'active' => false,
'action' => 'm', //m for migrate, s for synchronize
'active' => false
),
array( 'name' => 'with e class stuff',
'filename' => 'db_matches_2.php',
'connection' => $config2,
'active' => true,
'action' => 'm', //m for migrate, s for synchronize
'active' => true
),
);

65
tests/migrate/migrate.php
Unescape View File

@ -1,9 +1,5 @@
<?php
/**
* Base process migration script. This script reads a specific configuration file and contacts libraries
* in order to know what to migrate and where
* @package chamilo.migration
*/
/**
* Load required classes
*/
@ -13,7 +9,7 @@ require_once 'config.php';
require_once api_get_path(LIBRARY_PATH).'attendance.lib.php';
require_once api_get_path(LIBRARY_PATH).'thematic.lib.php';
error_reporting(-1);
//error_reporting(-1);
if (is_file(dirname(__FILE__) . '/migration.custom.class.php')) {
require_once 'migration.custom.class.php';
@ -25,21 +21,22 @@ require_once 'migration.class.php';
if (empty($servers)) {
die("This script requires a servers array with the connection settings and the file to parse\n");
}
/**
* Read the configuration file
*/
$start = time();
echo "\n-- Starting at ".date('h:i:s')." local server time\n";
if (!empty($servers)) {
foreach ($servers as $server_info) {
foreach($servers as $server_info) {
if ($server_info['active']) {
echo "\n---- Start loading server----- \n";
echo $server_info['name']."\n\n";
//echo "---- ----------------------- \n";
$config_info = $server_info['connection'];
$config_info = $server_info['connection'];
$db_type = $config_info['type'];
if (empty($db_type)) {
die("This script requires a DB type to work. Please update orig_db_conn.inc.php\n");
}
}
$file = dirname(__FILE__) . '/migration.' . $db_type . '.class.php';
if (!is_file($file)) {
die("Could not find db type file " . $file . "\n");
@ -48,29 +45,35 @@ if (!empty($servers)) {
$class = 'Migration' . strtoupper($db_type);
$m = new $class($config_info['host'], $config_info['port'], $config_info['db_user'], $config_info['db_pass'], $config_info['db_name'], $boost);
$m->connect();
if ($server_info['m']) {
/**
* Prepare the arrays of matches that will allow for the migration
*/
$migrate = array();
include $server_info['filename'];
//Default migration from MSSQL to Chamilo MySQL
$m->migrate($matches);
} elseif ($server_info['s']) {
//Getting transactions from MSSQL (via webservices)
if (isset($matches['web_service_calls']['filename'])) {
require_once $matches['web_service_calls']['filename'];
}
//This functions truncates the transaction lists!
$m->test_transactions($matches['web_service_calls']);
//$m->search_transactions($matches['web_service_calls']);
//Load transactions saved before
$m->load_transactions($matches);
/**
* Prepare the arrays of matches that will allow for the migration
*/
$migrate = array();
include $server_info['filename'];
//Default migration from MSSQL to Chamilo MySQL
$m->migrate($matches);
//Getting transactions from MSSQL (via webservices)
if (isset($matches['web_service_calls']['filename'])) {
require_once $matches['web_service_calls']['filename'];
}
//This functions truncates the transaction lists!
//$m->test_transactions($matches['web_service_calls']);
//$m->search_transactions($matches['web_service_calls']);
//Load transactions saved before
//$m->load_transactions($matches);
//print_r($m->errors_stack);
//echo "OK so far\n";
echo "\n ---- End loading server----- \n";
} else {
error_log("db_matches not activated: {$server_info['name']} {$server_info['filename']}");
}
}
}
}
$end = time();
echo "Total process took ".($end-$start)." seconds";

36
tests/migrate/migration.class.php
Unescape View File

@ -55,15 +55,14 @@ class Migration {
* data. Store values here (preferably using the same indexes as the
* destination database field names) until ready to insert into Chamilo.
*/
public $data_list = array('users' => array(), 'courses' => array(), 'sessions' => array());
/**
* Whether to use the memory instead of the database (if the memory allows)
* This option has to be configured manually, and the default is to use
* the database, which is slower but much safer in terms of stability of
* the script (or let's say much more linear in terms of execution time)
*/
private $boost = array('users' => false, 'courses' => false, 'sessions' => false);
public $data_list = array(
'boost_users' => false,
'users' => array(),
'boost_courses' => false,
'courses' => array(),
'boost_sessions' => false,
'sessions' => array(),
);
/**
* The constructor assigns all database connection details to the migration
* object
@ -72,7 +71,6 @@ class Migration {
* @param string The original database's user
* @param string The original database's password
* @param string The original database's name
* @param array Array of parameters to define the memory-boost behaviour
* @return boolean False on error. Void on success.
*/
public function __construct($dbhost, $dbport, $dbuser, $dbpass, $dbname, $boost=false) {
@ -88,7 +86,11 @@ class Migration {
$this->odbname = $dbname;
// Set the boost level if set in config.php
if (!empty($boost) && is_array($boost)) {
$this->boost = $boost;
foreach ($boost as $item => $val) {
if ($val == true) {
$this->data_list[$item] = true;
}
}
}
}
@ -105,11 +107,12 @@ class Migration {
* @param array Structured array of matches (see migrate.php)
*/
public function migrate($matches) {
error_log("\n" . '------------ Migration->migrate function called ------------' . "\n");
error_log("\n" . '------------ ['.date('H:i:s').'] Migration->migrate function called ------------' . "\n");
$extra_fields = array();
// Browsing through 1st-level arrays in db_matches.php
foreach ($matches as $table) {
error_log('Found table ' . $table['orig_table'] . ' in db_matches');
echo "Starting table ".$table['orig_table']." at ".date('h:i:s')."\n";
error_log('['.date('H:i:s').'] Found table ' . $table['orig_table'] . ' in db_matches');
$build_only = false;
if (empty($table['dest_table'])) {
@ -779,6 +782,7 @@ class Migration {
if ($details['func'] == 'none' || empty($details['func'])) {
$dest_data = $row[$details['orig']];
} else {
//error_log(__FILE__.' '.__LINE__.' Preparing to treat field with '.$details['func']);
$dest_data = MigrationCustom::$details['func']($row[$details['orig']], $this->data_list, $row);
}
@ -798,8 +802,10 @@ class Migration {
$field_type = $extra_field['field_type'];
if (!empty($options)) {
if (!is_array($options)) { $options = array($options); }
foreach ($options as $option) {
foreach ($option as $key => $value) {
if (is_array($option)) {
foreach ($option as $key => $value) {
//error_log("$key $value --> {$dest_row[$details['dest']]} ");
if ($key == 'option_value' && $value == $dest_row[$details['dest']]) {
$value = $option['option_display_text'];
@ -813,6 +819,7 @@ class Migration {
break(2);
}
}
}
}
}
} else {
@ -866,6 +873,7 @@ class Migration {
}
} else {
global $api_failureList;
error_log('Empty user details');
error_log(print_r($api_failureList, 1));
}
break;

51
tests/migrate/migration.custom.class.php
Unescape View File

@ -64,11 +64,16 @@ class MigrationCustom {
return utf8_encode($value);
}
static function clean_session_name($value, &$omigrate, $row_data) {
static function clean_session_name($value, $omigrate, $row_data) {
return self::clean_utf8($row_data['session_name']);
}
static function get_real_course_code($data) {
static function get_real_course_code($data, $omigrate=null) {
if (is_object($omigrate) && $omigrate->boost_courses) {
if (isset($omigrate->courses[$data])) {
return $omigrate->courses[$data];
}
}
$extra_field = new ExtraFieldValue('course');
$values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdCurso', $data);
if ($values) {
@ -78,7 +83,12 @@ class MigrationCustom {
}
}
static function get_session_id_by_programa_id($uidIdProgram) {
static function get_session_id_by_programa_id($uidIdProgram, $omigrate=null) {
if (is_object($omigrate) && $omigrate->boost_sessions) {
if (isset($omigrate->sessions[$uidIdPrograma])) {
return $omigrate->sessions[$uidIdPrograma];
}
}
$extra_field = new ExtraFieldValue('session');
$values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPrograma', $uidIdProgram);
if ($values) {
@ -89,7 +99,12 @@ class MigrationCustom {
}
/* Not used */
static function get_user_id_by_persona_id($uidIdPersona) {
static function get_user_id_by_persona_id($uidIdPersona, $omigrate=null) {
if (is_object($omigrate) && $omigrate->boost_users) {
if (isset($omigrate->users[$uidIdPersona])) {
return $omigrate->users[$uidIdPersona];
}
}
//error_log('get_user_id_by_persona_id');
$extra_field = new ExtraFieldValue('user');
$values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPersona', $uidIdPersona);
@ -100,12 +115,17 @@ class MigrationCustom {
}
}
static function get_real_teacher_id($uidIdPersona) {
static function get_real_teacher_id($uidIdPersona, $omigrate=null) {
$default_teacher_id = self::default_admin_id;
if (empty($uidIdPersona)) {
//error_log('No teacher provided');
return $default_teacher_id;
}
if (is_object($omigrate) && $omigrate->boost_users) {
if (isset($omigrate->users[$uidIdPersona])) {
return $omigrate->users[$uidIdPersona];
}
}
$extra_field = new ExtraFieldValue('user');
$values = $extra_field->get_item_id_from_field_variable_and_field_value('uidIdPersona', $uidIdPersona);
@ -244,7 +264,12 @@ class MigrationCustom {
//error_log(print_r($data, 1));
$user_info = UserManager::add($data);
if (!$user_info) {
echo 'error';
error_log('User '.$id_persona.' could not be inserted (maybe duplicate?)');
} else {
//error_log('User '.$id_persona.' was created as user '.$user_info['user_id']);
}
if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_users) {
$omigrate->users[$id_persona] = $user_info['user_id'];
}
UserManager::update_extra_field_value($user_info['user_id'], 'uidIdPersona', $id_persona);
return $user_info;
@ -253,7 +278,7 @@ class MigrationCustom {
/**
* Manages the course creation based on the rules in db_matches.php
*/
static function create_course($data) {
static function create_course($data, $omigrate) {
//error_log('In create_course, received '.print_r($data,1));
//Fixes wrong wanted codes
$data['wanted_code'] = str_replace(array('-', '_'), '000', $data['wanted_code']);
@ -272,6 +297,9 @@ class MigrationCustom {
'max' => '20'
);*/
$course_data = CourseManager::create_course($data);
if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_courses) {
$omigrate->courses[$data['uidIdCurso']] = $course_data['code'];
}
return $course_data;
}
@ -279,7 +307,7 @@ class MigrationCustom {
* Manages the session creation, based on data provided by the rules
* in db_matches.php
*/
static function create_session($data) {
static function create_session($data, $omigrate) {
//Hack to add the default gradebook course to the session course
$data['create_gradebook_evaluation'] = true;
/*$data['gradebook_params'] = array(
@ -298,7 +326,10 @@ class MigrationCustom {
//print_r($data);
//exit;
} else{
//error_log('session_id created');
//error_log('session_id created');
if (is_object($omigrate) && isset($omigrate) && $omigrate->boost_sessions) {
$omigrate->sessions[$data['uidIdPrograma']] = $session_id;
}
}
return $session_id;
}
@ -1188,4 +1219,4 @@ class MigrationCustom {
static function transaction_meses_editar($data, $web_service_details) {
self::transaction_extra_field_editar_generic('meses', $data, $web_service_details);
}
}
}

6
tests/migrate/migration.mssql.class.php
Unescape View File

@ -33,13 +33,13 @@ class MigrationMSSQL extends Migration {
//In order to process X item of each table add TOP X
$top = null;
//$top = " TOP 1000 ";
// $top = " TOP 100000 ";
if (in_array($table, array('Empleado', 'Alumno'))) {
$top = " TOP 10000 ";
// $top = " TOP 10000 ";
}
if (in_array($table, array('ProgramaAcademico', 'Matricula'))) {
$top = " TOP 100000 ";
// $top = " TOP 100000 ";
}
//$top = null;

Write Preview
Loading…
Cancel
Save