chamilo
/
chamilo-lms
mirror of https://github.com/chamilo/chamilo-lms/tree/1.11.x
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Adding more funcionality to the cron import csv process (dump all, add extra fields) see BT#6369

Browse Source
1.9.x
Julio Montoya 12 years ago
parent 2ce18b2131
commit fc452e9d3e
2 changed files with 287 additions and 54 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 290
      main/cron/import_csv.php
  2. 51
      main/inc/lib/sessionmanager.lib.php

290
main/cron/import_csv.php
Unescape View File

@ -7,15 +7,40 @@ if (PHP_SAPI!='cli') {
require_once __DIR__.'/../inc/global.inc.php';
require_once api_get_path(LIBRARY_PATH).'log.class.php';
/**
* Class ImportCsv
*/
class ImportCsv
{
private $logger;
private $dumpValues;
public $test;
public $defaultLanguage = 'english';
/**
* @param Logger $logger
*/
public function __construct($logger)
{
$this->logger = $logger;
}
/**
* @param bool $dump
*/
function setDumpValues($dump)
{
$this->dumpValues = $dump;
}
function getDumpValues()
{
return $this->dumpValues;
}
/**
* Runs the import process
*/
public function run()
{
$path = api_get_path(SYS_CODE_PATH).'cron/incoming/';
@ -24,34 +49,134 @@ class ImportCsv
exit;
}
if ($this->getDumpValues()) {
$table = Database::get_main_table(TABLE_MAIN_USER);
$sql = "DELETE FROM $table WHERE username NOT IN ('admin') AND lastname <> 'Anonymous' ";
Database::query($sql);
echo 'Dumping tables'.PHP_EOL;
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_COURSE);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_COURSE_USER);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_ACCESS_URL_REL_USER);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_SESSION);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_SESSION_CATEGORY);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_SESSION_COURSE);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
$table = Database::get_main_table(TABLE_MAIN_SESSION_USER);
$sql = "DELETE FROM $table";
Database::query($sql);
echo $sql.PHP_EOL;
}
echo "Starting with reading the files: ".PHP_EOL.PHP_EOL;
$files = scandir($path);
$fileToProcess = array();
if (!empty($files)) {
foreach ($files as $file) {
$fileInfo = pathinfo($file);
if ($fileInfo['extension'] == 'csv') {
// teachers_yyyymmdd.csv, courses_yyyymmdd.csv, students_yyyymmdd.csv and sessions_yyyymmdd.csv
$parts = explode('_', $fileInfo['filename']);
$method = 'import'.ucwords($parts[0]);
$method = 'import'.ucwords($parts[1]);
if (method_exists($this, $method)) {
$this->$method($path.$fileInfo['basename']);
$fileToProcess[$parts[1]][] = array(
'method' => $method,
'file' => $path.$fileInfo['basename']
);
//$this->$method($path.$fileInfo['basename']);
} else {
echo "Error - This file can't be processed.";
echo "Error - This file '$file' can't be processed.".PHP_EOL;
echo "The file have to has this format:".PHP_EOL;
echo "prefix_students_ddmmyyyy.csv, prefix_teachers_ddmmyyyy.csv, prefix_courses_ddmmyyyy.csv, prefix_sessions_ddmmyyyy.csv ".PHP_EOL;
exit;
}
}
}
if (empty($fileToProcess)) {
echo 'Error - no files to process.';
exit;
}
$sections = array('students', 'teachers', 'courses', 'sessions');
$this->prepareImport();
foreach ($sections as $section) {
$this->logger->addInfo("-- Import $section --");
if (isset($fileToProcess[$section]) && !empty($fileToProcess[$section])) {
$files = $fileToProcess[$section];
foreach ($files as $fileInfo) {
$method = $fileInfo['method'];
$file = $fileInfo['file'];
echo 'Reading file: '.$file.PHP_EOL;
$this->logger->addInfo("Reading file: $file");
$this->$method($file);
}
}
}
}
}
private function prepareImport()
{
// Create user extra field: extra_external_user_id
UserManager::create_extra_field('external_user_id', 1, 'External user id', null);
// Create course extra field: extra_external_course_id
CourseManager::create_course_extra_field('external_course_id', 1, 'External course id');
// Create session extra field extra_external_session_id
SessionManager::create_session_extra_field('external_session_id', 1, 'External session id');
}
/**
* @param string $file
*/
function moveFile($file)
{
$moved = str_replace('incoming', 'treated', $file);
// $result = rename($file, $moved);
$result = 1;
if ($this->test) {
$result = 1;
} else {
$result = rename($file, $moved);
}
if ($result) {
$this->logger->addInfo("Moving file to the treated folder: $file");
} else {
@ -59,15 +184,63 @@ class ImportCsv
}
}
/**
* @param array $row
*
* @return array
*/
private function cleanUserRow($row)
{
$row['lastname'] = $row['LastName'];
$row['firstname'] = $row['FirstName'];
$row['email'] = $row['Email'];
$row['username'] = $row['UserName'];
$row['password'] = $row['Password'];
$row['auth_srouce'] = $row['AuthSource'];
$row['official_code'] = $row['OfficialCode'];
$row['phone'] = $row['PhoneNumber'];
if (isset($row['StudentID'])) {
$row['extra_external_user_id'] = $row['StudentID'];
}
if (isset($row['TeacherID'])) {
$row['extra_external_user_id'] = $row['TeacherID'];
}
//$row['lastname'] = Status
return $row;
}
/**
* @param array $row
* @return array
*/
private function cleanCourseRow($row)
{
$row['title'] = $row['Title'];
$row['course_code'] = $row['Code'];
$row['category_code'] = $row['CourseCategory'];
$row['email'] = $row['Teacher'];
$row['language'] = $row['Language'];
$row['external_id'] = $row['CourseID'];
if (isset($row['CourseID'])) {
$row['extra_external_course_id'] = $row['CourseID'];
}
return $row;
}
/**
* File to import
* @param string $file
*/
function importTeachers($file)
private function importTeachers($file)
{
$data = Import::csv_to_array($file);
$this->logger->addInfo("-- Import Teachers --");
$this->logger->addInfo("Reading file: $file");
/* Unique identifier: official-code username.
Email address and password should never get updated. *ok
@ -77,15 +250,18 @@ class ImportCsv
We’ll handle that manually if applicable.
No delete!
*/
$language = $this->defaultLanguage;
if (!empty($data)) {
$this->logger->addInfo(count($data)." records found.");
foreach ($data as $row) {
$row = $this->cleanUserRow($row);
$userInfo = api_get_user_info_from_username($row['username']);
$userInfoByOfficialCode = api_get_user_info_from_official_code($row['official_code']);
if (empty($userInfo) && empty($userInfoByOfficialCode)) {
// Create user
$result = UserManager::create_user(
$userId = UserManager::create_user(
$row['firstname'],
$row['lastname'],
COURSEMANAGER,
@ -93,11 +269,11 @@ class ImportCsv
$row['username'],
$row['password'],
$row['official_code'],
$row['language'],
$language, //$row['language'],
$row['phone'],
$row['picture'], //picture
null, //$row['picture'], //picture
PLATFORM_AUTH_SOURCE, // ?
$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
'0000-00-00 00:00:00', //$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
1, //active
0,
null, // extra
@ -105,14 +281,19 @@ class ImportCsv
false //$send_mail = false
);
if ($result) {
$this->logger->addInfo("Info - Teachers - User created: ".$row['username']);
if ($userId) {
foreach ($row as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
UserManager::update_extra_field_value($userId, substr($key, 6), $value);
}
}
$this->logger->addInfo("Teachers - User created: ".$row['username']);
} else {
$this->logger->addError("Error - Teachers - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
$this->logger->addError("Teachers - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
}
} else {
if (empty($userInfo)) {
$this->logger->addError("Error - Teachers - Can't update user :".$row['username']);
$this->logger->addError("Teachers - Can't update user :".$row['username']);
continue;
}
@ -141,26 +322,27 @@ class ImportCsv
);
if ($result) {
foreach ($row as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
UserManager::update_extra_field_value($userInfo['user_id'], substr($key, 6), $value);
}
}
$this->logger->addInfo("Teachers - User updated: ".$row['username']);
} else {
$this->logger->addError("Teachers - User not updated: ".$row['username']);
}
}
// UserManager::delete_user();
$this->moveFile($file);
}
}
$this->moveFile($file);
}
/**
* @param string $file
*/
function importStudents($file)
private function importStudents($file)
{
$data = Import::csv_to_array($file);
$this->logger->addInfo("-- Import Students --");
$this->logger->addInfo("Reading file: $file");
/*
* Another users import.
@ -172,7 +354,10 @@ class ImportCsv
He should be set inactive one year after the current date. So I presume you’ll just update the expiration date. We want to grant access to courses up to a year after deletion.
*/
if (!empty($data)) {
$language = $this->defaultLanguage;
$this->logger->addInfo(count($data)." records found.");
foreach ($data as $row) {
$row = $this->cleanUserRow($row);
$userInfo = api_get_user_info_from_username($row['username']);
$userInfoByOfficialCode = api_get_user_info_from_official_code($row['official_code']);
@ -181,16 +366,16 @@ class ImportCsv
$result = UserManager::create_user(
$row['firstname'],
$row['lastname'],
STUDENT,
COURSEMANAGER,
$row['email'],
$row['username'],
$row['password'],
$row['official_code'],
$row['language'],
$language, //$row['language'],
$row['phone'],
$row['picture'], //picture
null, //$row['picture'], //picture
PLATFORM_AUTH_SOURCE, // ?
$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
'0000-00-00 00:00:00', //$row['expiration_date'], //$expiration_date = '0000-00-00 00:00:00',
1, //active
0,
null, // extra
@ -199,6 +384,11 @@ class ImportCsv
);
if ($result) {
foreach ($row as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
UserManager::update_extra_field_value($result, substr($key, 6), $value);
}
}
$this->logger->addInfo("Students - User created: ".$row['username']);
} else {
$this->logger->addError("Students - User NOT created: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
@ -239,14 +429,17 @@ class ImportCsv
);
if ($result) {
foreach ($row as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
UserManager::update_extra_field_value($userInfo['user_id'], substr($key, 6), $value);
}
}
$this->logger->addInfo("Students - User updated: ".$row['username']);
} else {
$this->logger->addError("Students - User NOT updated: ".$row['username']." ".$row['firstname']." ".$row['lastname']);
}
}
// UserManager::delete_user();
$this->moveFile($file);
}
}
@ -256,13 +449,16 @@ class ImportCsv
/**
* @param string $file
*/
function importCourses($file)
private function importCourses($file)
{
$data = Import::csv_to_array($file);
$this->logger->addInfo("Reading file: $file");
$language = $this->defaultLanguage;
if (!empty($data)) {
$this->logger->addInfo(count($data)." records found.");
foreach ($data as $row) {
$row = $this->cleanCourseRow($row);
$courseInfo = api_get_course_info($row['course_code']);
if (empty($courseInfo)) {
// Create
@ -276,6 +472,7 @@ class ImportCsv
$courseInfo = CourseManager::create_course($params);
if (!empty($courseInfo)) {
CourseManager::update_course_extra_field_value($courseInfo['code'], 'external_course_id', $row['extra_external_course_id']);
$this->logger->addInfo("Courses - Course created ".$courseInfo['code']);
} else {
$this->logger->addError("Courses - Can't create course:".$row['title']);
@ -294,18 +491,6 @@ class ImportCsv
} else {
$this->logger->addError("Courses - Course NOT updated ".$courseInfo['code']);
}
/*course_language='".Database::escape_string($course_language)."',
title='".Database::escape_string($title)."',
category_code='".Database::escape_string($category_code)."',
tutor_name='".Database::escape_string($tutor_name)."',
visual_code='".Database::escape_string($visual_code)."',
department_name='".Database::escape_string($department_name)."',
department_url='".Database::escape_string($department_url)."',
disk_quota='".Database::escape_string($disk_quota)."',
visibility = '".Database::escape_string($visibility)."',
subscribe = '".Database::escape_string($subscribe)."',
unsubscribe='*/
}
}
}
@ -315,9 +500,9 @@ class ImportCsv
/**
* @param string $file
*/
function importSessions($file)
private function importSessions($file)
{
$result = SessionManager::importCSV($file, true, 1, $this->logger);
$result = SessionManager::importCSV($file, true, 1, $this->logger, array('SessionId' => 'extra_external_session_id'));
if (!empty($result['error_message'])) {
$this->logger->addError($result['error_message']);
@ -326,6 +511,7 @@ class ImportCsv
$this->moveFile($file);
}
}
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Handler\NativeMailerHandler;
@ -345,8 +531,8 @@ $from = api_get_setting('emailAdministrator');
if (!empty($emails)) {
foreach ($emails as $email) {
$stream = new NativeMailerHandler($email, $subject, $from, $minLevel);
$logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
//$stream = new NativeMailerHandler($email, $subject, $from, $minLevel);
//$logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
}
}
@ -355,4 +541,14 @@ $logger->pushHandler(new BufferHandler($stream, 0, $minLevel));
$logger->pushHandler(new RotatingFileHandler('import_csv', 5, $minLevel));
$import = new ImportCsv($logger);
$dump = false;
if (isset($argv[1]) && $argv[1] = '--dump') {
$dump = true;
}
$import->setDumpValues($dump);
// Do not moves the files to treated
$import->test = true;
$import->run();

51
main/inc/lib/sessionmanager.lib.php
Unescape View File

@ -11,8 +11,13 @@
* Code
*/
class SessionManager {
private function __construct() {
/**
* Class SessionManager
*/
class SessionManager
{
private function __construct()
{
}
/**
@ -20,7 +25,8 @@ class SessionManager {
* @param int Session ID
* @return array Session details (id, id_coach, name, nbr_courses, nbr_users, nbr_classes, date_start, date_end, nb_days_access_before_beginning,nb_days_access_after_end, session_admin_id)
*/
public static function fetch($id) {
public static function fetch($id)
{
$t = Database::get_main_table(TABLE_MAIN_SESSION);
if ($id != strval(intval($id))) { return array(); }
$s = "SELECT * FROM $t WHERE id = $id";
@ -1863,11 +1869,14 @@ class SessionManager {
* false: if session exists a new session will be created adding a counter session1, session2, etc
* @param int $user_id
* @param $logger
* @param array convert a file row to an extra field. Example in CSV file there's a SessionID then it will
* converted to extra_external_session_id if you set this: array('SessionId' => 'extra_external_session_id')
* @return array
*/
static function importCSV($file, $updatesession, $user_id = null, $logger = null)
static function importCSV($file, $updatesession, $user_id = null, $logger = null, $extraFields = array())
{
$content = file($file);
$error_message = null;
$session_counter = 0;
@ -1885,12 +1894,15 @@ class SessionManager {
$tbl_session_course = Database::get_main_table(TABLE_MAIN_SESSION_COURSE);
$tbl_session_course_user = Database::get_main_table(TABLE_MAIN_SESSION_COURSE_USER);
$sessions = array();
if (!api_strstr($content[0], ';')) {
$error_message = get_lang('NotCSV');
} else {
$tag_names = array();
foreach ($content as $key => $enreg) {
$enreg = explode(';', trim($enreg));
if ($key) {
foreach ($tag_names as $tag_key => $tag_name) {
@ -1912,7 +1924,13 @@ class SessionManager {
$user_counter = 0;
$course_counter = 0;
$session_name = $enreg['SessionName'];
if (isset($extraFields) && !empty($extraFields)) {
foreach ($extraFields as $original => $to) {
$enreg[$to] = $enreg[$original];
}
}
$session_name = Database::escape_string($enreg['SessionName']);
$date_start = $enreg['DateStart'];
$date_end = $enreg['DateEnd'];
$visibility = $enreg['Visibility'];
@ -1938,7 +1956,7 @@ class SessionManager {
if ($i > 1) {
$suffix = ' - '.$i;
}
$sql = 'SELECT 1 FROM '.$tbl_session.' WHERE name="'.Database::escape_string($session_name.$suffix).'"';
$sql = 'SELECT 1 FROM '.$tbl_session.' WHERE name="'.$session_name.$suffix.'"';
$rs = Database::query($sql);
if (Database::result($rs, 0, 0)) {
@ -1951,7 +1969,7 @@ class SessionManager {
// Creating the session.
$sql_session = "INSERT IGNORE INTO $tbl_session SET
name = '".Database::escape_string($session_name)."',
name = '".$session_name."',
id_coach = '$coach_id',
date_start = '$date_start',
date_end = '$date_end',
@ -1963,6 +1981,13 @@ class SessionManager {
if ($debug) {
if ($session_id) {
foreach ($enreg as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
self::update_session_extra_field_value($session_id, substr($key, 6), $value);
}
}
$logger->addInfo("Sessions - Session created: #$session_id - $session_name");
} else {
$logger->addError("Sessions - Session NOT created: $session_name");
@ -1989,6 +2014,11 @@ class SessionManager {
if ($debug) {
if ($session_id) {
foreach ($enreg as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
self::update_session_extra_field_value($session_id, substr($key, 6), $value);
}
}
$logger->addInfo("Sessions - #$session_id created: $session_name");
} else {
$logger->addError("Sessions - Session NOT created: $session_name");
@ -2004,7 +2034,14 @@ class SessionManager {
session_category_id = '$session_category_id'
WHERE name = '$session_name'";
Database::query($sql_session);
$session_id = Database::query("SELECT id FROM $tbl_session WHERE name='$session_name'");
foreach ($enreg as $key => $value) {
if (substr($key, 0, 6) == 'extra_') { //an extra field
self::update_session_extra_field_value($session_id, substr($key, 6), $value);
}
}
list($session_id) = Database::fetch_array($session_id);
Database::query("DELETE FROM $tbl_session_user WHERE id_session='$session_id'");
Database::query("DELETE FROM $tbl_session_course WHERE id_session='$session_id'");

Write Preview
Loading…
Cancel
Save