Merge pull request #5934 from AngelFQC/BT21930

Plugin: Azure: Add options to user delta queries when syncing

plugin/azure_active_directory/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Azure Active Directory Changelog
 
+## 2.5 - 2024-11-18
+
+* Added new options to get the user and groups with delta query (or change tracking) when syncing with scripts.
+this requires manually doing the following changes to  your database if you are upgrading from v2.4
+```sql
+CREATE TABLE azure_ad_sync_state (id INT AUTO_INCREMENT NOT NULL, title VARCHAR(255) NOT NULL, value LONGTEXT NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY(id)) DEFAULT CHARACTER SET utf8 COLLATE `utf8_unicode_ci` ENGINE = InnoDB;
+```
+
 ## 2.4 - 2024-08-28
 
 * Added a new user extra field to save the unique Azure ID (internal UID).

plugin/azure_active_directory/lang/dutch.php

@@ -46,3 +46,7 @@ $strings['tenant_id'] = 'Mandanten-ID';
 $strings['tenant_id_help'] = 'Required to run scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Delta query for users';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Delta query for usergroups';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

plugin/azure_active_directory/lang/english.php

@@ -46,3 +46,7 @@ $strings['tenant_id'] = 'Tenant ID';
 $strings['tenant_id_help'] = 'Required to run scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Delta query for users';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Delta query for usergroups';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

plugin/azure_active_directory/lang/french.php

@@ -46,3 +46,7 @@ $strings['tenant_id'] = 'ID du client';
 $strings['tenant_id_help'] = 'Nécessaire pour exécuter des scripts.';
 $strings['deactivate_nonexisting_users'] = 'Deactivate non-existing users';
 $strings['deactivate_nonexisting_users_help'] = 'Compare registered users in Chamilo with those in Azure and deactivate accounts in Chamilo that do not exist in Azure.';
+$strings['script_users_delta'] = 'Requête delta pour les utilisateurs';
+$strings['script_users_delta_help'] = 'Get newly created, updated, or deleted users without having to perform a full read of the entire user collection. By default, is <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Requête delta pour les groupes d\'utilisateurs';
+$strings['script_usergroups_delta_help'] = 'Get newly created, updated, or deleted groups, including group membership changes, without having to perform a full read of the entire group collection. By default, is <code>No</code>.';

plugin/azure_active_directory/lang/spanish.php

@@ -46,3 +46,7 @@ $strings['tenant_id'] = 'Id. del inquilino';
 $strings['tenant_id_help'] = 'Necesario para ejecutar scripts.';
 $strings['deactivate_nonexisting_users'] = 'Desactivar usuarios no existentes';
 $strings['deactivate_nonexisting_users_help'] = 'Compara los usuarios registrados en Chamilo con los de Azure y desactiva las cuentas en Chamilo que no existan en Azure.';
+$strings['script_users_delta'] = 'Consula delta para usuarios';
+$strings['script_users_delta_help'] = 'Obtiene usuarios recién creados, actualizados o eliminados sin tener que realizar una lectura completa de toda la colección de usuarios. De forma predeterminada, es <code>No</code>.';
+$strings['script_usergroups_delta'] = 'Consulta delta para grupos de usuarios';
+$strings['script_usergroups_delta_help'] = 'Obtiene grupos recién creados, actualizados o eliminados, incluidos los cambios de membresía del grupo, sin tener que realizar una lectura completa de toda la colección de grupos. De forma predeterminada, es <code>No</code>';

plugin/azure_active_directory/src/AzureActiveDirectory.php

@@ -1,7 +1,10 @@
 <?php
 /* For license terms, see /license.txt */
 
+use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
 use Chamilo\UserBundle\Entity\User;
+use Doctrine\ORM\Tools\SchemaTool;
+use Doctrine\ORM\Tools\ToolsException;
 use TheNetworg\OAuth2\Client\Provider\Azure;
 
 /**
@@ -28,6 +31,8 @@ class AzureActiveDirectory extends Plugin
     public const SETTING_EXISTING_USER_VERIFICATION_ORDER = 'existing_user_verification_order';
     public const SETTING_TENANT_ID = 'tenant_id';
     public const SETTING_DEACTIVATE_NONEXISTING_USERS = 'deactivate_nonexisting_users';
+    public const SETTING_GET_USERS_DELTA = 'script_users_delta';
+    public const SETTING_GET_USERGROUPS_DELTA = 'script_usergroups_delta';
 
     public const URL_TYPE_AUTHORIZE = 'login';
     public const URL_TYPE_LOGOUT = 'logout';
@@ -59,9 +64,11 @@ class AzureActiveDirectory extends Plugin
             self::SETTING_EXISTING_USER_VERIFICATION_ORDER => 'text',
             self::SETTING_TENANT_ID => 'text',
             self::SETTING_DEACTIVATE_NONEXISTING_USERS => 'boolean',
+            self::SETTING_GET_USERS_DELTA => 'boolean',
+            self::SETTING_GET_USERGROUPS_DELTA => 'boolean',
         ];
 
-        parent::__construct('2.4', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
+        parent::__construct('2.5', 'Angel Fernando Quiroz Campos, Yannick Warnier', $settings);
     }
 
     /**
@@ -131,6 +138,8 @@ class AzureActiveDirectory extends Plugin
 
     /**
      * Create extra fields for user when installing.
+     *
+     * @throws ToolsException
      */
     public function install()
     {
@@ -152,6 +161,35 @@ class AzureActiveDirectory extends Plugin
             $this->get_lang('AzureUid'),
             ''
         );
+
+        $em = Database::getManager();
+
+        if ($em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
+            return;
+        }
+
+        $schemaTool = new SchemaTool($em);
+        $schemaTool->createSchema(
+            [
+                $em->getClassMetadata(AzureSyncState::class),
+            ]
+        );
+    }
+
+    public function uninstall()
+    {
+        $em = Database::getManager();
+
+        if (!$em->getConnection()->getSchemaManager()->tablesExist(['course_home_notify_notification'])) {
+            return;
+        }
+
+        $schemaTool = new SchemaTool($em);
+        $schemaTool->dropSchema(
+            [
+                $em->getClassMetadata(AzureSyncState::class),
+            ]
+        );
     }
 
     public function getExistingUserVerificationOrder(): array
@@ -385,4 +423,27 @@ class AzureActiveDirectory extends Plugin
             $extra,
         ];
     }
+
+    public function getSyncState(string $title): ?AzureSyncState
+    {
+        $stateRepo = Database::getManager()->getRepository(AzureSyncState::class);
+
+        return $stateRepo->findOneBy(['title' => $title]);
+    }
+
+    public function saveSyncState(string $title, $value)
+    {
+        $state = $this->getSyncState($title);
+
+        if (!$state) {
+            $state = new AzureSyncState();
+            $state->setTitle($title);
+
+            Database::getManager()->persist($state);
+        }
+
+        $state->setValue($value);
+
+        Database::getManager()->flush();
+    }
 }

plugin/azure_active_directory/src/AzureCommand.php

@@ -2,6 +2,7 @@
 
 /* For license terms, see /license.txt */
 
+use Chamilo\PluginBundle\Entity\AzureActiveDirectory\AzureSyncState;
 use League\OAuth2\Client\Provider\Exception\IdentityProviderException;
 use League\OAuth2\Client\Token\AccessTokenInterface;
 use TheNetworg\OAuth2\Client\Provider\Azure;
@@ -56,11 +57,21 @@ abstract class AzureCommand
             'id',
         ];
 
-        $query = sprintf(
-            '$top=%d&$select=%s',
-            AzureActiveDirectory::API_PAGE_SIZE,
-            implode(',', $userFields)
-        );
+        $getUsersDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA);
+
+        if ($getUsersDelta) {
+            $usersDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERS_DATALINK);
+
+            $query = $usersDeltaLink
+                ? $usersDeltaLink->getValue()
+                : sprintf('$select=%s', implode(',', $userFields));
+        } else {
+            $query = sprintf(
+                '$top=%d&$select=%s',
+                AzureActiveDirectory::API_PAGE_SIZE,
+                implode(',', $userFields)
+            );
+        }
 
         $token = null;
 
@@ -70,7 +81,7 @@ abstract class AzureCommand
             try {
                 $azureUsersRequest = $this->provider->request(
                     'get',
-                    "users?$query",
+                    $getUsersDelta ? "users/delta?$query" : "users?$query",
                     $token
                 );
             } catch (Exception $e) {
@@ -80,6 +91,10 @@ abstract class AzureCommand
             $azureUsersInfo = $azureUsersRequest['value'] ?? [];
 
             foreach ($azureUsersInfo as $azureUserInfo) {
+                $azureUserInfo['mail'] = $azureUserInfo['mail'] ?? null;
+                $azureUserInfo['surname'] = $azureUserInfo['surname'] ?? null;
+                $azureUserInfo['givenName'] = $azureUserInfo['givenName'] ?? null;
+
                 yield $azureUserInfo;
             }
 
@@ -89,6 +104,13 @@ abstract class AzureCommand
                 $hasNextLink = true;
                 $query = parse_url($azureUsersRequest['@odata.nextLink'], PHP_URL_QUERY);
             }
+
+            if ($getUsersDelta && !empty($azureUsersRequest['@odata.deltaLink'])) {
+                $this->plugin->saveSyncState(
+                    AzureSyncState::USERS_DATALINK,
+                    parse_url($azureUsersRequest['@odata.deltaLink'], PHP_URL_QUERY),
+                );
+            }
         } while ($hasNextLink);
     }
 
@@ -105,11 +127,21 @@ abstract class AzureCommand
             'description',
         ];
 
-        $query = sprintf(
-            '$top=%d&$select=%s',
-            AzureActiveDirectory::API_PAGE_SIZE,
-            implode(',', $groupFields)
-        );
+        $getUsergroupsDelta = 'true' === $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERGROUPS_DELTA);
+
+        if ($getUsergroupsDelta) {
+            $usergroupsDeltaLink = $this->plugin->getSyncState(AzureSyncState::USERGROUPS_DATALINK);
+
+            $query = $usergroupsDeltaLink
+                ? $usergroupsDeltaLink->getValue()
+                : sprintf('$select=%s', implode(',', $groupFields));
+        } else {
+            $query = sprintf(
+                '$top=%d&$select=%s',
+                AzureActiveDirectory::API_PAGE_SIZE,
+                implode(',', $groupFields)
+            );
+        }
 
         $token = null;
 
@@ -117,7 +149,11 @@ abstract class AzureCommand
             $this->generateOrRefreshToken($token);
 
             try {
-                $azureGroupsRequest = $this->provider->request('get', "groups?$query", $token);
+                $azureGroupsRequest = $this->provider->request(
+                    'get',
+                    $getUsergroupsDelta ? "groups/delta?$query" : "groups?$query",
+                    $token
+                );
             } catch (Exception $e) {
                 throw new Exception('Exception when requesting groups from Azure: '.$e->getMessage());
             }
@@ -134,6 +170,13 @@ abstract class AzureCommand
                 $hasNextLink = true;
                 $query = parse_url($azureGroupsRequest['@odata.nextLink'], PHP_URL_QUERY);
             }
+
+            if ($getUsergroupsDelta && !empty($azureGroupsRequest['@odata.deltaLink'])) {
+                $this->plugin->saveSyncState(
+                    AzureSyncState::USERGROUPS_DATALINK,
+                    parse_url($azureGroupsRequest['@odata.deltaLink'], PHP_URL_QUERY),
+                );
+            }
         } while ($hasNextLink);
     }
 

plugin/azure_active_directory/src/AzureSyncUsersCommand.php

@@ -15,8 +15,8 @@ class AzureSyncUsersCommand extends AzureCommand
     {
         yield 'Synchronizing users from Azure.';
 
-        /** @var array<string, int> $existingUsers */
-        $existingUsers = [];
+        /** @var array<string, int> $azureCreatedUserIdList */
+        $azureCreatedUserIdList = [];
 
         foreach ($this->getAzureUsers() as $azureUserInfo) {
             try {
@@ -27,7 +27,7 @@ class AzureSyncUsersCommand extends AzureCommand
                 continue;
             }
 
-            $existingUsers[$azureUserInfo['id']] = $userId;
+            $azureCreatedUserIdList[$azureUserInfo['id']] = $userId;
 
             yield sprintf('User (ID %d) with received info: %s ', $userId, serialize($azureUserInfo));
         }
@@ -53,7 +53,7 @@ class AzureSyncUsersCommand extends AzureCommand
             $azureGroupMembersUids = array_column($azureGroupMembersInfo, 'id');
 
             foreach ($azureGroupMembersUids as $azureGroupMembersUid) {
-                $userId = $existingUsers[$azureGroupMembersUid] ?? null;
+                $userId = $azureCreatedUserIdList[$azureGroupMembersUid] ?? null;
 
                 if (!$userId) {
                     continue;
@@ -72,20 +72,22 @@ class AzureSyncUsersCommand extends AzureCommand
             $em->flush();
         }
 
-        if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)) {
+        if ('true' === $this->plugin->get(AzureActiveDirectory::SETTING_DEACTIVATE_NONEXISTING_USERS)
+            && 'true' !== $this->plugin->get(AzureActiveDirectory::SETTING_GET_USERS_DELTA)
+        ) {
             yield '----------------';
 
             yield 'Trying deactivate non-existing users in Azure';
 
             $users = UserManager::getRepository()->findByAuthSource('azure');
-            $userIdList = array_map(
+            $chamiloUserIdList = array_map(
                 function ($user) {
                     return $user->getId();
                 },
                 $users
             );
 
-            $nonExistingUsers = array_diff($userIdList, $existingUsers);
+            $nonExistingUsers = array_diff($chamiloUserIdList, $azureCreatedUserIdList);
 
             UserManager::deactivate_users($nonExistingUsers);
 

plugin/azure_active_directory/src/Entity/AzureSyncState.php

@@ -0,0 +1,74 @@
+<?php
+
+/* For licensing terms, see /license.txt */
+
+namespace Chamilo\PluginBundle\Entity\AzureActiveDirectory;
+
+use Chamilo\CoreBundle\Traits\TimestampableTypedEntity;
+use Doctrine\ORM\Mapping as ORM;
+
+/**
+ * @package Chamilo\PluginBundle\Entity\AzureActiveDirectory
+ *
+ * @ORM\Table(name="azure_ad_sync_state")
+ * @ORM\Entity()
+ */
+class AzureSyncState
+{
+    use TimestampableTypedEntity;
+
+    public const USERS_DATALINK = 'users_datalink';
+    public const USERGROUPS_DATALINK = 'usergroups_datalink';
+
+    /**
+     * @var int
+     *
+     * @ORM\Column(name="id", type="integer")
+     * @ORM\Id()
+     * @ORM\GeneratedValue()
+     */
+    private int $id = 0;
+
+    /**
+     * @var string
+     *
+     * @ORM\Column(name="title", type="string")
+     */
+    private string $title;
+
+    /**
+     * @var string
+     *
+     * @ORM\Column(name="value", type="text")
+     */
+    private string $value;
+
+    public function getId(): int
+    {
+        return $this->id;
+    }
+
+    public function getTitle(): string
+    {
+        return $this->title;
+    }
+
+    public function setTitle(string $title): AzureSyncState
+    {
+        $this->title = $title;
+
+        return $this;
+    }
+
+    public function getValue(): string
+    {
+        return $this->value;
+    }
+
+    public function setValue(string $value): AzureSyncState
+    {
+        $this->value = $value;
+
+        return $this;
+    }
+}

plugin/azure_active_directory/uninstall.php

@@ -0,0 +1,9 @@
+<?php
+
+/* For licensing terms, see /license.txt */
+
+if (!api_is_platform_admin()) {
+    exit('You must have admin permissions to uninstall plugins');
+}
+
+AzureActiveDirectory::create()->uninstall();