From f8352fcb8dfb5284520396bdc624b6bf62d18219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 19 Oct 2016 10:03:29 +0200 Subject: [PATCH] introduce callForSeenUsers and countSeenUsers (#26361) * introduce callForSeenUsers and countSeenUsers * add tests * oracle should support not null on clob * since 9.2.0 --- .../lib/BackgroundJob/ExpireTrash.php | 4 +- .../lib/Command/ExpireTrash.php | 4 +- .../lib/BackgroundJob/ExpireVersions.php | 4 +- .../lib/Command/ExpireVersions.php | 4 +- lib/private/Repair/RemoveRootShares.php | 21 +-- lib/private/Repair/RepairUnmergedShares.php | 19 +-- lib/private/User/Manager.php | 135 +++++++++++++++--- lib/public/IUserManager.php | 15 ++ tests/lib/Repair/RemoveRootSharesTest.php | 3 + tests/lib/User/ManagerTest.php | 64 ++++++++- 10 files changed, 208 insertions(+), 65 deletions(-) diff --git a/apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php b/apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php index 7d481afeb59..d366a401e2f 100644 --- a/apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php +++ b/apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php @@ -77,9 +77,9 @@ class ExpireTrash extends \OC\BackgroundJob\TimedJob { return; } - $this->userManager->callForAllUsers(function(IUser $user) { + $this->userManager->callForSeenUsers(function(IUser $user) { $uid = $user->getUID(); - if ($user->getLastLogin() === 0 || !$this->setupFS($uid)) { + if (!$this->setupFS($uid)) { return; } $dirContent = Helper::getTrashFiles('/', $uid, 'mtime'); diff --git a/apps/files_trashbin/lib/Command/ExpireTrash.php b/apps/files_trashbin/lib/Command/ExpireTrash.php index ff827718885..b016fdd6aeb 100644 --- a/apps/files_trashbin/lib/Command/ExpireTrash.php +++ b/apps/files_trashbin/lib/Command/ExpireTrash.php @@ -89,7 +89,7 @@ class ExpireTrash extends Command { } else { $p = new ProgressBar($output); $p->start(); - $this->userManager->callForAllUsers(function(IUser $user) use ($p) { + $this->userManager->callForSeenUsers(function(IUser $user) use ($p) { $p->advance(); $this->expireTrashForUser($user); }); @@ -100,7 +100,7 @@ class ExpireTrash extends Command { function expireTrashForUser(IUser $user) { $uid = $user->getUID(); - if ($user->getLastLogin() === 0 || !$this->setupFS($uid)) { + if (!$this->setupFS($uid)) { return; } $dirContent = Helper::getTrashFiles('/', $uid, 'mtime'); diff --git a/apps/files_versions/lib/BackgroundJob/ExpireVersions.php b/apps/files_versions/lib/BackgroundJob/ExpireVersions.php index 8e1f02cdfbf..89b8a966137 100644 --- a/apps/files_versions/lib/BackgroundJob/ExpireVersions.php +++ b/apps/files_versions/lib/BackgroundJob/ExpireVersions.php @@ -67,9 +67,9 @@ class ExpireVersions extends \OC\BackgroundJob\TimedJob { return; } - $this->userManager->callForAllUsers(function(IUser $user) { + $this->userManager->callForSeenUsers(function(IUser $user) { $uid = $user->getUID(); - if ($user->getLastLogin() === 0 || !$this->setupFS($uid)) { + if (!$this->setupFS($uid)) { return; } Storage::expireOlderThanMaxForUser($uid); diff --git a/apps/files_versions/lib/Command/ExpireVersions.php b/apps/files_versions/lib/Command/ExpireVersions.php index f384420f22f..e88ea1f7a02 100644 --- a/apps/files_versions/lib/Command/ExpireVersions.php +++ b/apps/files_versions/lib/Command/ExpireVersions.php @@ -88,7 +88,7 @@ class ExpireVersions extends Command { } else { $p = new ProgressBar($output); $p->start(); - $this->userManager->callForAllUsers(function(IUser $user) use ($p) { + $this->userManager->callForSeenUsers(function(IUser $user) use ($p) { $p->advance(); $this->expireVersionsForUser($user); }); @@ -99,7 +99,7 @@ class ExpireVersions extends Command { function expireVersionsForUser(IUser $user) { $uid = $user->getUID(); - if ($user->getLastLogin() === 0 || !$this->setupFS($uid)) { + if (!$this->setupFS($uid)) { return; } Storage::expireOlderThanMaxForUser($uid); diff --git a/lib/private/Repair/RemoveRootShares.php b/lib/private/Repair/RemoveRootShares.php index 1fd7d8d7dae..69fcb1b4492 100644 --- a/lib/private/Repair/RemoveRootShares.php +++ b/lib/private/Repair/RemoveRootShares.php @@ -96,30 +96,13 @@ class RemoveRootShares implements IRepairStep { $output->advance(); }; - $userCount = $this->countUsers(); - $output->startProgress($userCount); + $output->startProgress($this->userManager->countSeenUsers()); - $this->userManager->callForAllUsers($function); + $this->userManager->callForSeenUsers($function); $output->finishProgress(); } - /** - * Count all the users - * - * @return int - */ - private function countUsers() { - $allCount = $this->userManager->countUsers(); - - $totalCount = 0; - foreach ($allCount as $backend => $count) { - $totalCount += $count; - } - - return $totalCount; - } - /** * Verify if this repair steps is required * It *should* not be necessary in most cases and it can be very diff --git a/lib/private/Repair/RepairUnmergedShares.php b/lib/private/Repair/RepairUnmergedShares.php index d57bc3779f8..56d935c74f5 100644 --- a/lib/private/Repair/RepairUnmergedShares.php +++ b/lib/private/Repair/RepairUnmergedShares.php @@ -335,22 +335,6 @@ class RepairUnmergedShares implements IRepairStep { } } - /** - * Count all the users - * - * @return int - */ - private function countUsers() { - $allCount = $this->userManager->countUsers(); - - $totalCount = 0; - foreach ($allCount as $backend => $count) { - $totalCount += $count; - } - - return $totalCount; - } - public function run(IOutput $output) { $ocVersionFromBeforeUpdate = $this->config->getSystemValue('version', '0.0.0'); if (version_compare($ocVersionFromBeforeUpdate, '9.1.0.16', '<')) { @@ -363,8 +347,7 @@ class RepairUnmergedShares implements IRepairStep { $this->buildPreparedQueries(); - $userCount = $this->countUsers(); - $output->startProgress($userCount); + $output->startProgress($this->userManager->countUsers()); $this->userManager->callForAllUsers($function); diff --git a/lib/private/User/Manager.php b/lib/private/User/Manager.php index 7d8c6d48b2c..f0488637485 100644 --- a/lib/private/User/Manager.php +++ b/lib/private/User/Manager.php @@ -314,10 +314,16 @@ class Manager extends PublicEmitter implements IUserManager { /** * returns how many users per backend exist (if supported by backend) * - * @return array an array of backend class as key and count number as value + * @param boolean $hasLoggedIn when true only users that have a lastLogin + * entry in the preferences table will be affected + * @return array|int an array of backend class as key and count number as value + * if $hasLoggedIn is true only an int is returned */ - public function countUsers() { - $userCountStatistics = array(); + public function countUsers($hasLoggedIn = false) { + if ($hasLoggedIn) { + return $this->countSeenUsers(); + } + $userCountStatistics = []; foreach ($this->backends as $backend) { if ($backend->implementsActions(Backend::COUNT_USERS)) { $backendUsers = $backend->countUsers(); @@ -344,29 +350,120 @@ class Manager extends PublicEmitter implements IUserManager { * * @param \Closure $callback * @param string $search + * @param boolean $onlySeen when true only users that have a lastLogin entry + * in the preferences table will be affected * @since 9.0.0 */ - public function callForAllUsers(\Closure $callback, $search = '') { - foreach($this->getBackends() as $backend) { - $limit = 500; - $offset = 0; - do { - $users = $backend->getUsers($search, $limit, $offset); - foreach ($users as $uid) { - if (!$backend->userExists($uid)) { - continue; + public function callForAllUsers(\Closure $callback, $search = '', $onlySeen = false) { + if ($onlySeen) { + $this->callForSeenUsers($callback); + } else { + foreach ($this->getBackends() as $backend) { + $limit = 500; + $offset = 0; + do { + $users = $backend->getUsers($search, $limit, $offset); + foreach ($users as $uid) { + if (!$backend->userExists($uid)) { + continue; + } + $user = $this->getUserObject($uid, $backend, false); + $return = $callback($user); + if ($return === false) { + break; + } } - $user = $this->getUserObject($uid, $backend, false); - $return = $callback($user); - if ($return === false) { - break; + $offset += $limit; + } while (count($users) >= $limit); + } + } + } + + /** + * returns how many users have logged in once + * + * @return int + * @since 9.2.0 + */ + public function countSeenUsers() { + $queryBuilder = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $queryBuilder->select($queryBuilder->createFunction('COUNT(*)')) + ->from('preferences') + ->where($queryBuilder->expr()->eq( + 'appid', $queryBuilder->createNamedParameter('login')) + ) + ->andWhere($queryBuilder->expr()->eq( + 'configkey', $queryBuilder->createNamedParameter('lastLogin')) + ) + ->andWhere($queryBuilder->expr()->isNotNull('configvalue') + ); + + $query = $queryBuilder->execute(); + return (int)$query->fetchColumn(); + } + + /** + * @param \Closure $callback + * @param string $search + * @since 9.2.0 + */ + public function callForSeenUsers (\Closure $callback) { + $limit = 1000; + $offset = 0; + do { + $userIds = $this->getSeenUserIds($limit, $offset); + $offset += $limit; + foreach ($userIds as $userId) { + foreach ($this->backends as $backend) { + if ($backend->userExists($userId)) { + $user = $this->getUserObject($userId, $backend, false); + $return = $callback($user); + if ($return === false) { + return; + } } } - $offset += $limit; - } while (count($users) >= $limit); - } + } + } while (count($userIds) >= $limit); } + /** + * Getting all userIds that have a listLogin value requires checking the + * value in php because on oracle you cannot use a clob in a where clause, + * preventing us from doing a not null or length(value) > 0 check. + * + * @param int $limit + * @param int $offset + * @return string[] with user ids + */ + private function getSeenUserIds($limit = null, $offset = null) { + $queryBuilder = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $queryBuilder->select(['userid']) + ->from('preferences') + ->where($queryBuilder->expr()->eq( + 'appid', $queryBuilder->createNamedParameter('login')) + ) + ->andWhere($queryBuilder->expr()->eq( + 'configkey', $queryBuilder->createNamedParameter('lastLogin')) + ) + ->andWhere($queryBuilder->expr()->isNotNull('configvalue') + ); + + if ($limit !== null) { + $queryBuilder->setMaxResults($limit); + } + if ($offset !== null) { + $queryBuilder->setFirstResult($offset); + } + $query = $queryBuilder->execute(); + $result = []; + + while ($row = $query->fetch()) { + $result[] = $row['userid']; + } + + return $result; + } /** * @param string $email * @return IUser[] diff --git a/lib/public/IUserManager.php b/lib/public/IUserManager.php index 1e0c298edcf..854622335c8 100644 --- a/lib/public/IUserManager.php +++ b/lib/public/IUserManager.php @@ -144,6 +144,21 @@ interface IUserManager { */ public function callForAllUsers (\Closure $callback, $search = ''); + /** + * returns how many users have logged in once + * + * @return int + * @since 9.2.0 + */ + public function countSeenUsers(); + + /** + * @param \Closure $callback + * @param string $search + * @since 9.2.0 + */ + public function callForSeenUsers (\Closure $callback); + /** * @param string $email * @return IUser[] diff --git a/tests/lib/Repair/RemoveRootSharesTest.php b/tests/lib/Repair/RemoveRootSharesTest.php index bf255fc7e9b..cfb81cb1ecc 100644 --- a/tests/lib/Repair/RemoveRootSharesTest.php +++ b/tests/lib/Repair/RemoveRootSharesTest.php @@ -106,6 +106,7 @@ class RemoveRootSharesTest extends \Test\TestCase { $user = $this->userManager->createUser('test', 'test'); $userFolder = $this->rootFolder->getUserFolder('test'); $fileId = $userFolder->getId(); + $user->updateLastLoginTimestamp(); //Now insert cyclic share $qb = $this->connection->getQueryBuilder(); @@ -134,6 +135,7 @@ class RemoveRootSharesTest extends \Test\TestCase { $user1 = $this->userManager->createUser('test1', 'test1'); $userFolder = $this->rootFolder->getUserFolder('test1'); $fileId = $userFolder->getId(); + $user1->updateLastLoginTimestamp(); //Now insert cyclic share $qb = $this->connection->getQueryBuilder(); @@ -156,6 +158,7 @@ class RemoveRootSharesTest extends \Test\TestCase { $userFolder = $this->rootFolder->getUserFolder('test2'); $folder = $userFolder->newFolder('foo'); $fileId = $folder->getId(); + $user2->updateLastLoginTimestamp(); //Now insert cyclic share $qb = $this->connection->getQueryBuilder(); diff --git a/tests/lib/User/ManagerTest.php b/tests/lib/User/ManagerTest.php index f6318ff353b..8cad7f4241a 100644 --- a/tests/lib/User/ManagerTest.php +++ b/tests/lib/User/ManagerTest.php @@ -9,6 +9,8 @@ namespace Test\User; use OC\User\Database; +use OCP\IUser; +use Test\TestCase; /** * Class ManagerTest @@ -17,7 +19,7 @@ use OC\User\Database; * * @package Test\User */ -class ManagerTest extends \Test\TestCase { +class ManagerTest extends TestCase { public function testGetBackends() { $userDummyBackend = $this->createMock(\Test\Util\User\Dummy::class); $manager = new \OC\User\Manager(); @@ -448,6 +450,66 @@ class ManagerTest extends \Test\TestCase { $this->assertEquals(7 + 16, $users); } + public function testCountUsersOnlySeen() { + $manager = \OC::$server->getUserManager(); + // count other users in the db before adding our own + $countBefore = $manager->countUsers(true); + + //Add test users + $user1 = $manager->createUser('testseencount1', 'testseencount1'); + $user1->updateLastLoginTimestamp(); + + $user2 = $manager->createUser('testseencount2', 'testseencount2'); + $user2->updateLastLoginTimestamp(); + + $user3 = $manager->createUser('testseencount3', 'testseencount3'); + + $user4 = $manager->createUser('testseencount4', 'testseencount4'); + $user4->updateLastLoginTimestamp(); + + $this->assertEquals($countBefore + 3, $manager->countUsers(true)); + + //cleanup + $user1->delete(); + $user2->delete(); + $user3->delete(); + $user4->delete(); + } + + public function testCallForSeenUsers() { + $manager = \OC::$server->getUserManager(); + // count other users in the db before adding our own + $count = 0; + $function = function (IUser $user) use (&$count) { + $count++; + }; + $manager->callForAllUsers($function, '', true); + $countBefore = $count; + + //Add test users + $user1 = $manager->createUser('testseen1', 'testseen1'); + $user1->updateLastLoginTimestamp(); + + $user2 = $manager->createUser('testseen2', 'testseen2'); + $user2->updateLastLoginTimestamp(); + + $user3 = $manager->createUser('testseen3', 'testseen3'); + + $user4 = $manager->createUser('testseen4', 'testseen4'); + $user4->updateLastLoginTimestamp(); + + $count = 0; + $manager->callForAllUsers($function, '', true); + + $this->assertEquals($countBefore + 3, $count); + + //cleanup + $user1->delete(); + $user2->delete(); + $user3->delete(); + $user4->delete(); + } + public function testDeleteUser() { $config = $this->getMockBuilder('OCP\IConfig') ->disableOriginalConstructor()