open-dsi
/
nextcloud-server
mirror of https://github.com/nextcloud/server
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #48200 from nextcloud/share-reminder-sharding

Browse Source 
fix: add sharding compatible version of share reminder job
pull/45471/head
Robin Appelman 7 months ago committed by GitHub
parent 086b11f8fb 0bdd25eb0f
commit 21678df4d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 104 additions and 13 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 113
      apps/files_sharing/lib/SharesReminderJob.php
  2. 4
      apps/files_sharing/tests/SharesReminderJobTest.php

113
apps/files_sharing/lib/SharesReminderJob.php
Unescape View File

@ -13,6 +13,8 @@ use OCP\Constants;
use OCP\DB\Exception;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Defaults;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
use OCP\IDBConnection;
use OCP\IL10N;
@ -33,6 +35,7 @@ use Psr\Log\LoggerInterface;
class SharesReminderJob extends TimedJob {
private const SECONDS_BEFORE_REMINDER = 86400;
private const CHUNK_SIZE = 1000;
private int $folderMimeTypeId;
public function __construct(
ITimeFactory $time,
@ -44,9 +47,11 @@ class SharesReminderJob extends TimedJob {
private readonly IFactory $l10nFactory,
private readonly IMailer $mailer,
private readonly Defaults $defaults,
IMimeTypeLoader $mimeTypeLoader,
) {
parent::__construct($time);
$this->setInterval(3600);
$this->folderMimeTypeId = $mimeTypeLoader->getId(ICacheEntry::DIRECTORY_MIMETYPE);
}
@ -74,6 +79,30 @@ class SharesReminderJob extends TimedJob {
* @throws Exception if a database error occurs
*/
private function getShares(): array|\Iterator {
if ($this->db->getShardDefinition('filecache')) {
$sharesResult = $this->getSharesDataSharded();
} else {
$sharesResult = $this->getSharesData();
}
foreach($sharesResult as $share) {
if ($share['share_type'] === IShare::TYPE_EMAIL) {
$id = "ocMailShare:$share[id]";
} else {
$id = "ocinternal:$share[id]";
}
try {
yield $this->shareManager->getShareById($id);
} catch (ShareNotFound) {
$this->logger->error("Share with ID $id not found.");
}
}
}
/**
* @return list<array{id: int, share_type: int}>
*/
private function getSharesData(): array {
$minDate = new \DateTime();
$maxDate = new \DateTime();
$maxDate->setTimestamp($maxDate->getTimestamp() + self::SECONDS_BEFORE_REMINDER);
@ -103,21 +132,81 @@ class SharesReminderJob extends TimedJob {
)
->setMaxResults(SharesReminderJob::CHUNK_SIZE);
$sharesResult = $qb->executeQuery();
while ($share = $sharesResult->fetch()) {
if ((int)$share['share_type'] === IShare::TYPE_EMAIL) {
$id = "ocMailShare:$share[id]";
} else {
$id = "ocinternal:$share[id]";
}
$shares = $qb->executeQuery()->fetchAll();
return array_values(array_map(fn ($share): array => [
'id' => (int)$share['id'],
'share_type' => (int)$share['share_type'],
], $shares));
}
try {
yield $this->shareManager->getShareById($id);
} catch (ShareNotFound) {
$this->logger->error("Share with ID $id not found.");
/**
* Sharding compatible version of getSharesData
*
* @return list<array{id: int, share_type: int, file_source: int}>
*/
private function getSharesDataSharded(): array|\Iterator {
$minDate = new \DateTime();
$maxDate = new \DateTime();
$maxDate->setTimestamp($maxDate->getTimestamp() + self::SECONDS_BEFORE_REMINDER);
$qb = $this->db->getQueryBuilder();
$qb->select('s.id', 's.share_type', 's.file_source')
->from('share', 's')
->where(
$qb->expr()->andX(
$qb->expr()->orX(
$qb->expr()->eq('s.share_type', $qb->expr()->literal(IShare::TYPE_USER)),
$qb->expr()->eq('s.share_type', $qb->expr()->literal(IShare::TYPE_EMAIL))
),
$qb->expr()->eq('s.item_type', $qb->expr()->literal('folder')),
$qb->expr()->gte('s.expiration', $qb->createNamedParameter($minDate, IQueryBuilder::PARAM_DATE)),
$qb->expr()->lte('s.expiration', $qb->createNamedParameter($maxDate, IQueryBuilder::PARAM_DATE)),
$qb->expr()->eq('s.reminder_sent', $qb->createNamedParameter(
false, IQueryBuilder::PARAM_BOOL
)),
$qb->expr()->eq(
$qb->expr()->bitwiseAnd('s.permissions', Constants::PERMISSION_CREATE),
$qb->createNamedParameter(Constants::PERMISSION_CREATE, IQueryBuilder::PARAM_INT)
),
)
);
$shares = $qb->executeQuery()->fetchAll();
$shares = array_values(array_map(fn ($share): array => [
'id' => (int)$share['id'],
'share_type' => (int)$share['share_type'],
'file_source' => (int)$share['file_source'],
], $shares));
return $this->filterSharesWithEmptyFolders($shares, self::CHUNK_SIZE);
}
/**
* Check which of the supplied file ids is an empty folder until there are `$maxResults` folders
* @param list<array{id: int, share_type: int, file_source: int}> $shares
* @return list<array{id: int, share_type: int, file_source: int}>
*/
private function filterSharesWithEmptyFolders(array $shares, int $maxResults): array {
$query = $this->db->getQueryBuilder();
$query->select('fileid')
->from('filecache')
->where($query->expr()->eq('size', $query->createNamedParameter(0), IQueryBuilder::PARAM_INT_ARRAY))
->andWhere($query->expr()->eq('mimetype', $query->createNamedParameter($this->folderMimeTypeId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->in('fileid', $query->createParameter('fileids')));
$chunks = array_chunk($shares, SharesReminderJob::CHUNK_SIZE);
$results = [];
foreach ($chunks as $chunk) {
$chunkFileIds = array_map(fn ($share): int => $share['file_source'], $chunk);
$chunkByFileId = array_combine($chunkFileIds, $chunk);
$query->setParameter('fileids', $chunkFileIds, IQueryBuilder::PARAM_INT_ARRAY);
$chunkResults = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
foreach ($chunkResults as $folderId) {
$results[] = $chunkByFileId[$folderId];
}
if (count($results) >= $maxResults) {
break;
}
}
$sharesResult->closeCursor();
return $results;
}
/**

4
apps/files_sharing/tests/SharesReminderJobTest.php
Unescape View File

@ -12,6 +12,7 @@ use OCA\Files_Sharing\SharesReminderJob;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\Constants;
use OCP\Defaults;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\IRootFolder;
use OCP\IDBConnection;
use OCP\IURLGenerator;
@ -71,6 +72,7 @@ class SharesReminderJobTest extends \Test\TestCase {
\OC::$server->get(IFactory::class),
$this->mailer,
\OC::$server->get(Defaults::class),
\OC::$server->get(IMimeTypeLoader::class),
);
}
@ -158,7 +160,7 @@ class SharesReminderJobTest extends \Test\TestCase {
$testFolder = $user1Folder->newFolder('test');
if (!$isEmpty) {
$testFolder->newFile('some_file.txt');
$testFolder->newFile('some_file.txt', 'content');
}
$share = $this->shareManager->newShare();

Write Preview
Loading…
Cancel
Save