Internal: Improve user creation and update logic for CSV sync script - refs BT#21895

public/main/inc/global.inc.php

@@ -142,5 +142,5 @@ try {
     define('DEFAULT_DOCUMENT_QUOTA', 100000000);
 } catch (Exception $e) {
     $controller = new ExceptionController();
-    $controller->showAction($e);
+    $controller->show($e);
 }

public/main/inc/lib/import.lib.php

@@ -41,7 +41,7 @@ class Import
      *
      * @return array returns an array (in the system encoding) that contains all data from the CSV-file
      */
-    public static function csvToArray($filename)
+    public static function csvToArray($filename, $delimiter = ';'): array
     {
         if (empty($filename)) {
             return [];
@@ -49,15 +49,7 @@ class Import
 
         $reader = Reader::createFromPath($filename, 'r');
         if ($reader) {
-            $reader->setDelimiter(';');
+            $reader->setDelimiter($delimiter);
-            //$reader->stripBom(true);
-            /*$contents = $reader->__toString();
-            if (!Utf8::isUtf8($contents)) {
-                // If file is not in utf8 try converting to ISO-8859-15
-                if ($reader->getStreamFilterMode() == 1) {
-                    $reader->appendStreamFilter('convert.iconv.ISO-8859-15/UTF-8');
-                }
-            }*/
             $reader->setHeaderOffset(0);
             $iterator = $reader->getRecords();
 

src/CoreBundle/EventListener/LegacyListener.php

@@ -148,9 +148,9 @@ class LegacyListener
             $session->set('cid_reset', false);
         }
 
-        $session->set(
+        $currentAccessUrl = $this->accessUrlHelper->getCurrent();
-            'access_url_id',
+        if (null !== $currentAccessUrl) {
-            $this->accessUrlHelper->getCurrent()->getId()
+            $session->set('access_url_id', $currentAccessUrl->getId());
-        );
+        }
     }
 }

src/CoreBundle/ServiceHelper/AccessUrlHelper.php

@@ -29,14 +29,14 @@ class AccessUrlHelper
         return $accessUrlEnabled;
     }
 
-    public function getFirstAccessUrl(): AccessUrl
+    public function getFirstAccessUrl(): ?AccessUrl
     {
         $urlId = $this->accessUrlRepository->getFirstId();
 
-        return $this->accessUrlRepository->find($urlId);
+        return $this->accessUrlRepository->find($urlId) ?: null;
     }
 
-    public function getCurrent(): AccessUrl
+    public function getCurrent(): ?AccessUrl
     {
         static $accessUrl;
 

src/CoreBundle/ServiceHelper/ThemeHelper.php

@@ -39,6 +39,10 @@ final class ThemeHelper
      */
     public function getVisualTheme(): string
     {
+        if ('cli' === PHP_SAPI) {
+            return '';
+        }
+
         static $visualTheme;
 
         global $lp_theme_css;

tests/scripts/synchronize_user_base_from_csv.php

@@ -20,55 +20,63 @@ username field is used to identify and match CSV and Chamilo accounts together.
 */
 exit;
 // Change this to the absolute path to chamilo root folder if you move the script out of tests/scripts
-$chamiloRoot = __DIR__.'/../..';
+$chamiloRoot = __DIR__.'/../../public';
 
 // Set to true in order to get a trace of changes made by this script
-$debug = false;
+$debug = true;
 
-// Set to test mode by default to only show the output, put this test variable to 0 to enable creation, modificaction and deletion of users
+// Set to test mode by default to only show the output, put this test variable to 0 to enable creation, modificaction y deletion of users
-$test = 1;
+$test = 0;
 
 // It defines if the user not found in any of the CSV files but present in Chamilo should be deleted or disabled. By default it will be disabled.
 // Set it to true for users to be deleted.
 $deleteUsersNotFoundInCSV = false;
 
-// Re-enable users found in CSV file and that where present but inactivated in Chamilo
+// Re-enable users found in CSV file and that were present but inactivated in Chamilo
 $reenableUsersFoundInCSV = false;
 
 // Anonymize user accounts disabled for more than 3 years
 $anonymizeUserAccountsDisbaledFor3Years = false;
 
 // List of username of accounts that should not be disabled or deleted if not present in CSV
-// For exemple the first admin and the anonymous user that has no username ('')
+// For example the first admin and the anonymous user that has no username ('')
 //$usernameListNotToTouchEvenIfNotInCSV = ['admin','','test'];
 
-// Extra field to be emptied when user is anonimized to really make it anonyme, for example the sso id of the user
+// Extra field to be emptied when user is anonymized to really make it anonymous, for example the sso id of the user
-// extraFieldToEmpty = "cas_user";
+// $extraFieldToEmpty = "cas_user";
-
 
+use Chamilo\CoreBundle\Entity\Admin;
 use Chamilo\CoreBundle\Entity\ExtraFieldValues;
 use Chamilo\CoreBundle\Entity\ExtraField;
 use Chamilo\CoreBundle\Entity\TrackEDefault;
-use Chamilo\UserBundle\Entity\User;
+use Chamilo\CoreBundle\Entity\User;
-use Doctrine\DBAL\FetchMode;
 use Doctrine\ORM\OptimisticLockException;
 
 if (php_sapi_name() !== 'cli') {
     die("this script is supposed to be run from the command-line\n");
 }
 
-require $chamiloRoot.'/cli-config.php';
+require_once $chamiloRoot.'/main/inc/global.inc.php';
 require_once $chamiloRoot.'/main/inc/lib/api.lib.php';
 require_once $chamiloRoot.'/main/inc/lib/database.constants.inc.php';
 
 ini_set('memory_limit', -1);
 
+$statusList = [
+    'teacher' => 1,        // COURSEMANAGER
+    'session_admin' => 3,  // SESSIONADMIN
+    'drh' => 4,            // DRH
+    'user' => 5,           // STUDENT
+    'anonymous' => 6,      // ANONYMOUS
+    'invited' => 20        // INVITEE
+];
+
+$entityManager = Database::getManager();
 $allCSVUsers = [];
 const EXTRA_KEY = 'extra_';
 
-// read all users from the internal database
+// Read all users from the internal database
-
+$userRepository = $entityManager->getRepository(User::class);
-$userRepository = Database::getManager()->getRepository('ChamiloUserBundle:User');
 $dbUsers = [];
 foreach ($userRepository->findAll() as $user) {
     if ($user->getId() > 1) {
@@ -81,158 +89,281 @@ if ($debug) {
     echo count($dbUsers) . " users with id > 1 found in internal database\n";
 }
 
-if (api_is_multiple_url_enabled()) {
+$adminRepo = $entityManager->getRepository(Admin::class);
-    $accessUrls = api_get_access_urls(0,100000,'id');
+$firstAdmin = $adminRepo->createQueryBuilder('a')
-}
+    ->setMaxResults(1)
+    ->getQuery()
+    ->getOneOrNullResult();
 
-if ($debug) {
+if ($firstAdmin) {
-    echo "accessUrls = " . print_r($accessUrls,1);
+    $creator = $firstAdmin->getUser();
+} else {
+    die("No admin found in the database\n");
 }
 
-$allCSVUsers = [];
+$accessUrls = api_get_access_urls(0, 100000, 'id');
 foreach ($accessUrls as $accessUrl) {
     $accessUrlId = $accessUrl['id'];
-    // Read the content of the csv file for this url (file name is URLID_users.csv so for $accessUrlId = 0 the file name would be 0_users.csv
+    $filename = $chamiloRoot . "/../tests/scripts/" . $accessUrlId . "_usersexample.csv";
-    $filename = $accessUrlId . "_users.csv"
-    $CSVUsers = Import :: csvToArray($filename);
 
-    // create new user accounts found in the CSV and update the existing ones, re-enabling if necessary
+    if (!file_exists($filename)) {
+        if ($debug) {
+            echo "CSV file not found: $filename\n";
+        }
+        continue;
+    }
+
+    $CSVUsers = Import::csvToArray($filename, ',');
+
+    if (!$CSVUsers) {
+        die("Failed to parse CSV file: $filename\n");
+    }
+
+    // Debug message
+    echo "Processing file: $filename with " . count($CSVUsers) . " users\n";
+
+    $fieldMapping = [
+        'username' => 'setUsername',
+        'lastname' => 'setLastname',
+        'firstname' => 'setFirstname',
+        'email' => 'setEmail',
+        'officialcode' => 'setOfficialCode',
+        'phonenumber' => 'setPhone',
+        'status' => 'setStatus',
+        'expirydate' => 'setExpirationDate',
+        'active' => 'setActive',
+        'language' => 'setLocale',
+        'password' => 'setPlainPassword'
+    ];
+
+    // Create new user accounts found in the CSV and update the existing ones, re-enabling if necessary
     foreach ($CSVUsers as $CSVuser) {
-        if (empty($CSVuser['username']) {
+
-	    continue;
+        try {
-	}
+
-	$username = $CSVuser['username'];
+            $CSVuser = array_change_key_case($CSVuser); // Convert keys to lowercase
-        if (array_key_exists($username, $dbUsers)) {
+
-            $user = $dbUsers[$username];
+            if (empty($CSVuser['username'])) {
-            if ($debug) {
+                echo "Skipping user with empty username\n";
-                echo "User in DB = " . $username . " and user id = " . $user->getId() . "\n";
+                continue;
             }
-        } else {
+            $username = strtolower($CSVuser['username']);
-            if (!$test) {
+            if (array_key_exists($username, $dbUsers)) {
-                $user = new User();
+                $user = $dbUsers[$username];
-                $dbUsers[$username] = $user;
+                if ($debug) {
-                $user->setUsernameCanonical($username);
+                    echo "User in DB = " . $username . " and user id = " . $user->getId() . "\n";
+                }
+            } else {
+                if (!$test) {
+                    $user = new User();
+                    $dbUsers[$username] = $user;
+                    $user->setUsername($username);
+                    $user->setUsernameCanonical($username);
+                }
+                if ($debug) {
+                    echo 'Created ' . $username . "\n";
+                    echo "CSVUser = " . print_r($CSVuser, 1) . "\n";
+                }
             }
             if ($debug) {
-                echo 'Created ' . $username . "\n";
+                echo 'Updating ' . $username . ' fields ' . "\n";
-                echo "CSVUser = " . print_r($CSVUser,1) . "\n";
             }
-        }
-        if ($debug) {
-            echo 'Updating ' . $username . ' fields '."\n";
-        }
-        if (!$test) {
-            foreach ($CSVuser as $fieldValue => $fieldName) {
-                // verify if it's an extra field or not (if it contains EXTRA_KEY at the begining of the name)
 
-                // update every field and extra field of the user
+            if (!$test) {
+                $passwordSet = false;
+                foreach ($CSVuser as $fieldName => $fieldValue) {
+                    if (isset($fieldMapping[$fieldName])) {
+                        $setter = $fieldMapping[$fieldName];
+                        if ($setter === 'setExpirationDate') {
+                            $fieldValue = new DateTime($fieldValue);
+                        }
+                        if ($setter === 'setPlainPassword') {
+                            $passwordSet = true;
+                        }
+                        if ($setter === 'setStatus') {
+                            if (isset($statusList[$fieldValue])) {
+                                $fieldValue = $statusList[$fieldValue];
+                                $user->setRoleFromStatus($fieldValue);
+                            } else {
+                                die("Status value '$fieldValue' not found in status list\n");
+                            }
+                        }
+                        if (method_exists($user, $setter)) {
+                            $user->$setter($fieldValue);
+                        } else {
+                            die("Setter method '$setter' not found in User entity\n");
+                        }
+                    }
+                }
+
+                if (!$passwordSet) {
+                    $user->setPlainPassword(api_generate_password());
+                }
 
-                if (!$user->isActive() and $reenableUsersFoundInLDAP) {
+                if (!$user->isActive() && $reenableUsersFoundInCSV) {
                     $user->setActive(true);
                 }
-                Database::getManager()->persist($user);
+
+                $user->setCreator($creator);
+                $userRepository->updateUser($user, true);
+
+                foreach ($CSVuser as $fieldName => $fieldValue) {
+                    if (strpos($fieldName, EXTRA_KEY) === 0) {
+                        $extraFieldName = substr($fieldName, strlen(EXTRA_KEY));
+                        $extraField = $entityManager->getRepository(ExtraField::class)->findOneBy(['variable' => $extraFieldName]);
+                        if ($extraField) {
+                            $extraFieldValue = $entityManager->getRepository(ExtraFieldValues::class)->findOneBy(['field' => $extraField, 'itemId' => $user->getId()]);
+                            if (!$extraFieldValue) {
+                                $extraFieldValue = new ExtraFieldValues();
+                                $extraFieldValue->setField($extraField);
+                                $extraFieldValue->setItemId($user->getId());
+                            }
+                            $extraFieldValue->setFieldValue($fieldValue);
+                            $entityManager->persist($extraFieldValue);
+                        } else {
+                            die("Extra field '$extraFieldName' not found in database\n");
+                        }
+                    }
+                }
+
                 try {
-                    Database::getManager()->flush();
+                    $entityManager->flush();
-                } catch (OptimisticLockException $exception) {
+                } catch (OptimisticLockException $e) {
-                    die($exception->getMessage()."\n");
+                    echo "Error processing user '{$username}': " . $e->getMessage() . "\n";
+                    error_log("Error processing user '{$username}': " . $e->getMessage());
+                    echo "Trace: " . $e->getTraceAsString() . "\n";
+                    continue;
                 }
-                if($debug) {
+
+                if ($debug) {
                     echo 'Sent to DB ' . $username . " with user id = " . $user->getId() . "\n";
                 }
                 UrlManager::add_user_to_url($user->getId(), $accessUrlId);
             }
+
+            $allCSVUsers[$username] = $user;
+
+        } catch (Exception $e) {
+            echo "Error processing user '{$username}': " . $e->getMessage() . "\n";
+            error_log("Error processing user '{$username}': " . $e->getMessage());
+            echo "Trace: " . $e->getTraceAsString() . "\n";
+            continue;
         }
-        $allCSVUsers[$username] = $user;
     }
 }
 
-// disable or delete user accounts not found in any CSV file depending on $deleteUsersNotFoundInLDAP
+// Disable or delete user accounts not found in any CSV file depending on $deleteUsersNotFoundInCSV
-
 $now = new DateTime();
 foreach (array_diff(array_keys($dbUsers), array_keys($allCSVUsers)) as $usernameToDisable) {
-    if (in_array($usernameToDisable, $usernameListNotToTouchEvenIfNotInCSV)) {
+    if (isset($usernameListNotToTouchEvenIfNotInCSV) && in_array($usernameToDisable, $usernameListNotToTouchEvenIfNotInCSV)) {
         if ($debug) {
-            echo 'User not modified even if not present in LDAP : ' . $usernameToDisable . "\n";
+            echo 'User not modified even if not present in CSV: ' . $usernameToDisable . "\n";
         }
     } else {
         $user = $dbUsers[$usernameToDisable];
-        if ($deleteUsersNotFoundInLDAP) {
+        if ($deleteUsersNotFoundInCSV) {
             if (!$test) {
                 if (!UserManager::delete_user($user->getId())) {
                     if ($debug) {
                         echo 'Unable to delete user ' . $usernameToDisable . "\n";
                     }
+                } else {
+                    if ($debug) {
+                        echo 'Deleted user ' . $usernameToDisable . "\n";
+                    }
+                }
+            } else {
+                if ($debug) {
+                    echo 'Test mode: User ' . $usernameToDisable . ' would have been deleted\n';
                 }
-            }
-            if ($debug) {
-                echo 'Deleted user ' . $usernameToDisable . "\n";
             }
         } else {
             if (!$test) {
                 if ($user->isActive()) {
-                    // In order to avoid slow individual SQL updates, we do not call
-                    // UserManager::disable($user->getId());
                     $user->setActive(false);
-                    Database::getManager()->persist($user);
+                    $entityManager->persist($user);
-                    // In order to avoid slow individual SQL updates, we do not call
+
-                    // Event::addEvent(LOG_USER_DISABLE, LOG_USER_ID, $user->getId());
                     $trackEDefault = new TrackEDefault();
-                    $trackEDefault->setDefaultUserId(1);
+                    $trackEDefault->setDefaultUserId($firstAdmin->getId());
                     $trackEDefault->setDefaultDate($now);
                     $trackEDefault->setDefaultEventType(LOG_USER_DISABLE);
                     $trackEDefault->setDefaultValueType(LOG_USER_ID);
-                    $trackEDefault->setDefaultValue($user->getId());
+                    $trackEDefault->setDefaultValue((string) $user->getId());
-                    Database::getManager()->persist($trackEDefault);
+                    $entityManager->persist($trackEDefault);
+
+                    try {
+                        $entityManager->flush();
+                    } catch (OptimisticLockException $e) {
+                        error_log("Error processing user " . $e->getMessage());
+                        echo "Trace: " . $e->getTraceAsString() . "\n";
+                        continue;
+                    }
+
+                    if ($debug) {
+                        echo 'Disabled user ' . $usernameToDisable . "\n";
+                    }
+                } else {
+                    if ($debug) {
+                        echo 'User ' . $usernameToDisable . ' is already disabled\n';
+                    }
+                }
+            } else {
+                if ($debug) {
+                    echo 'Test mode: User ' . $usernameToDisable . ' would have been disabled\n';
                 }
-            }
-            if ($debug) {
-                echo 'Disabled ' . $user->getUsername() . "\n";
             }
         }
     }
 }
+
 if (!$test) {
     try {
-        // Saving everything together
+        $entityManager->flush();
-        Database::getManager()->flush();
+    } catch (OptimisticLockException $e) {
-    } catch (OptimisticLockException $exception) {
+        error_log("Error processing user " . $e->getMessage());
-        die($exception->getMessage()."\n");
+        echo "Trace: " . $e->getTraceAsString() . "\n";
     }
 }
 
-
+// Anonymize user accounts disabled for more than 3 years
-// anonymize user accounts disabled for more than 3 years
 if ($anonymizeUserAccountsDisbaledFor3Years) {
-    $longDisabledUserIds = [];
+    echo "Anonymizing user accounts disabled for more than 3 years\n";
-    foreach (Database::query(
+
-        'select default_value
+    $longDisabledUserIds = $entityManager->createQueryBuilder()
-        from track_e_default
+        ->select('t.defaultValue')
-        where default_event_type=\'user_disable\' and default_value_type=\'user_id\'
+        ->from(TrackEDefault::class, 't')
-        group by default_value
+        ->where('t.defaultEventType = :eventType')
-        having max(default_date) < date_sub(now(), interval 3 year)'
+        ->andWhere('t.defaultValueType = :valueType')
-    )->fetchAll(FetchMode::COLUMN) as $userId) {
+        ->andWhere('t.defaultDate < :date')
-        $longDisabledUserIds[] = $userId;
+        ->groupBy('t.defaultValue')
-    }
+        ->setParameter('eventType', 'user_disable')
-    $anonymizedUserIds = [];
+        ->setParameter('valueType', 'user_id')
-    foreach (Database::query(
+        ->setParameter('date', (new DateTime())->modify('-3 years'))
-        'select distinct default_value
+        ->getQuery()
-        from track_e_default
+        ->getSingleColumnResult();
-        where default_event_type=\'user_anonymized\' and default_value_type=\'user_id\''
+
-    )->fetchAll(FetchMode::COLUMN) as $userId) {
+    $anonymizedUserIds = $entityManager->createQueryBuilder()
-        $anonymizedUserIds[] = $userId;
+        ->select('t.defaultValue')
-    }
+        ->from(TrackEDefault::class, 't')
+        ->where('t.defaultEventType = :eventType')
+        ->andWhere('t.defaultValueType = :valueType')
+        ->distinct()
+        ->setParameter('eventType', 'user_anonymized')
+        ->setParameter('valueType', 'user_id')
+        ->getQuery()
+        ->getSingleColumnResult();
+
     foreach (array_diff($longDisabledUserIds, $anonymizedUserIds) as $userId) {
         $user = $userRepository->find($userId);
         if ($user && !$user->isEnabled()) {
             if (!$test) {
                 try {
-                    UserManager::anonymize($userId)
+                    UserManager::anonymize($userId) or die("could not anonymize user $userId\n");
-                    or die("could not anonymize user $userId\n");
                 } catch (Exception $exception) {
-                    die($exception->getMessage()."\n");
+                    die($exception->getMessage() . "\n");
-		}
+                }
                 if (isset($extraFieldToEmpty)) {
-                    UserManager::update_extra_field_value($userId,$extraFieldToEmpty,'');
+                    UserManager::update_extra_field_value($userId, $extraFieldToEmpty, '');
                 }
             }
             if ($debug) {