From 7100c7116676ce3de0dcd9d1dee467503b455d2b Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 26 Aug 2025 17:54:13 +0200 Subject: [PATCH] perf(metadata): Add optimized sharding for metadata deletion Signed-off-by: Carl Schwan --- lib/private/Files/Cache/Cache.php | 36 +++++++++++++------ .../FilesMetadata/FilesMetadataManager.php | 4 +-- .../FilesMetadata/Listener/MetadataDelete.php | 9 +++-- .../Service/MetadataRequestService.php | 6 ++-- .../Files/Cache/CacheEntriesRemovedEvent.php | 10 +++--- .../Files/Cache/CacheEntryRemovedEvent.php | 2 +- .../FilesMetadata/IFilesMetadataManager.php | 3 +- 7 files changed, 45 insertions(+), 25 deletions(-) diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index 3b7bdb5cc4d..6213b88fc17 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -635,18 +635,32 @@ class Cache implements ICache { } $cacheEntryRemovedEvents = []; - foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { - $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( - $this->storage, - $filePath, - $fileId, - $this->getNumericStorageId() - ); - $cacheEntryRemovedEvents[] = $cacheEntryRemovedEvent; - $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); - } - $this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents)); + foreach (array_chunk(array_combine($deletedIds, $deletedPaths), 1000) as $chunk) { + /** @var array $chunk */ + foreach ($chunk as $fileId => $filePath) { + $cacheEntryRemovedEvents[] = new CacheEntryRemovedEvent( + $this->storage, + $filePath, + $fileId, + $this->getNumericStorageId() + ); + } + $exception = null; + try { + $this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents)); + } catch (\Exception $e) { + // still send the other event + $exception = $e; + } + foreach ($cacheEntryRemovedEvents as $cacheEntryRemovedEvent) { + $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); + } + + if ($exception !== null) { + throw $exception; + } + } } /** diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index 55f534aedfb..188cb09e53a 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -214,9 +214,9 @@ class FilesMetadataManager implements IFilesMetadataManager { } } - public function deleteMetadataForFiles(array $fileIds): void { + public function deleteMetadataForFiles(int $storage, array $fileIds): void { try { - $this->metadataRequestService->dropMetadataForFiles($fileIds); + $this->metadataRequestService->dropMetadataForFiles($storage, $fileIds); } catch (Exception $e) { $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]); } diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php index 2c83cb259f6..5a404614df0 100644 --- a/lib/private/FilesMetadata/Listener/MetadataDelete.php +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -33,18 +33,21 @@ class MetadataDelete implements IEventListener { } $entries = $event->getCacheEntryRemovedEvents(); - $fileIds = []; + $storageToFileIds = []; foreach ($entries as $entry) { try { - $fileIds[] = $entry->getFileId(); + $storageToFileIds[$entry->getStorageId()] ??= []; + $storageToFileIds[$entry->getStorageId()][] = $entry->getFileId(); } catch (Exception $e) { $this->logger->warning('issue while running MetadataDelete', ['exception' => $e]); } } try { - $this->filesMetadataManager->deleteMetadataForFiles($fileIds); + foreach ($storageToFileIds as $storageId => $fileIds) { + $this->filesMetadataManager->deleteMetadataForFiles($storageId, $fileIds); + } } catch (Exception $e) { $this->logger->warning('issue while running MetadataDelete', ['exception' => $e]); } diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php index 68b08760627..c274c70812d 100644 --- a/lib/private/FilesMetadata/Service/MetadataRequestService.php +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -147,16 +147,16 @@ class MetadataRequestService { /** * @param int[] $fileIds - * @return void * @throws Exception */ - public function dropMetadataForFiles(array $fileIds): void { + public function dropMetadataForFiles(int $storage, array $fileIds): void { $chunks = array_chunk($fileIds, 1000); foreach ($chunks as $chunk) { $qb = $this->dbConnection->getQueryBuilder(); $qb->delete(self::TABLE_METADATA) - ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->hintShardKey('storage', $storage); $qb->executeStatement(); } } diff --git a/lib/public/Files/Cache/CacheEntriesRemovedEvent.php b/lib/public/Files/Cache/CacheEntriesRemovedEvent.php index fee6a5996e3..6038c3ce4a4 100644 --- a/lib/public/Files/Cache/CacheEntriesRemovedEvent.php +++ b/lib/public/Files/Cache/CacheEntriesRemovedEvent.php @@ -8,26 +8,28 @@ declare(strict_types=1); */ namespace OCP\Files\Cache; +use OCP\AppFramework\Attribute\Listenable; use OCP\EventDispatcher\Event; /** * Meta-event wrapping multiple CacheEntryRemovedEvent for when an existing * entry in the cache gets removed. * - * @since 32.0.0 + * @since 34.0.0 */ -#[\OCP\AppFramework\Attribute\Listenable(since: '32.0.0')] +#[Listenable(since: '34.0.0')] class CacheEntriesRemovedEvent extends Event { /** - * @param CacheEntryRemovedEvent[] $cacheEntryRemovedEvents + * @param ICacheEvent[] $cacheEntryRemovedEvents */ public function __construct( private readonly array $cacheEntryRemovedEvents, ) { + Event::__construct(); } /** - * @return CacheEntryRemovedEvent[] + * @return ICacheEvent[] */ public function getCacheEntryRemovedEvents(): array { return $this->cacheEntryRemovedEvents; diff --git a/lib/public/Files/Cache/CacheEntryRemovedEvent.php b/lib/public/Files/Cache/CacheEntryRemovedEvent.php index 8c4fed2b084..d015f3ba94d 100644 --- a/lib/public/Files/Cache/CacheEntryRemovedEvent.php +++ b/lib/public/Files/Cache/CacheEntryRemovedEvent.php @@ -11,7 +11,7 @@ namespace OCP\Files\Cache; /** * Event for when an existing entry in the cache gets removed * - * Prefer using \c CacheEntriesRemovedEvent as it is more efficient when deleting + * Prefer using CacheEntriesRemovedEvent as it is more efficient when deleting * multiple files at the same time. * * @since 21.0.0 diff --git a/lib/public/FilesMetadata/IFilesMetadataManager.php b/lib/public/FilesMetadata/IFilesMetadataManager.php index 55a2a3d8dc2..df16b9e3fde 100644 --- a/lib/public/FilesMetadata/IFilesMetadataManager.php +++ b/lib/public/FilesMetadata/IFilesMetadataManager.php @@ -103,11 +103,12 @@ interface IFilesMetadataManager { /** * Delete metadata and its indexes of multiple file ids * + * @param int $storage The storage id coresponding to the $fileIds * @param array $fileIds file ids * @return void * @since 32.0.0 */ - public function deleteMetadataForFiles(array $fileIds): void; + public function deleteMetadataForFiles(int $storage, array $fileIds): void; /** * generate and return a MetadataQuery to help building sql queries