From c192331dc05aac2dd7146db9d15330a20bd7d48f Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 28 Jul 2026 10:36:26 +0200 Subject: [PATCH] perf(preview): Optimize retriving all previews from oc_filecache Previoulsy we used a PATH LIKE expression which is fine for small instances but doesn't scale for big instance (timeout). Manually moving accross the tree with getFolderContentsById is significantly faster as we can use the index and this also reuse common APIs from OCP/Files instead of directly manipulating the filecache with the query builder. Assisted-by: ClaudeCode:claude-opus-4-8 Signed-off-by: Carl Schwan --- core/BackgroundJobs/PreviewMigrationJob.php | 103 +++++++----------- tests/lib/Preview/PreviewMigrationJobTest.php | 3 - 2 files changed, 41 insertions(+), 65 deletions(-) diff --git a/core/BackgroundJobs/PreviewMigrationJob.php b/core/BackgroundJobs/PreviewMigrationJob.php index 5ca3de9286a..9a9e64b9703 100644 --- a/core/BackgroundJobs/PreviewMigrationJob.php +++ b/core/BackgroundJobs/PreviewMigrationJob.php @@ -12,11 +12,10 @@ namespace OC\Core\BackgroundJobs; use OC\Preview\PreviewMigrationService; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; -use OCP\DB\IResult; +use OCP\Files\FileInfo; use OCP\Files\IRootFolder; use OCP\IAppConfig; use OCP\IConfig; -use OCP\IDBConnection; use Override; use Psr\Log\LoggerInterface; @@ -27,7 +26,6 @@ class PreviewMigrationJob extends TimedJob { ITimeFactory $time, private readonly IAppConfig $appConfig, private readonly IConfig $config, - private readonly IDBConnection $connection, private readonly IRootFolder $rootFolder, private readonly PreviewMigrationService $migrationService, private readonly LoggerInterface $logger, @@ -45,79 +43,60 @@ class PreviewMigrationJob extends TimedJob { return; } - $startTime = time(); - while (true) { - $qb = $this->connection->getQueryBuilder(); - $qb->select('path') - ->from('filecache') - ->where($qb->expr()->orX( - // Hierarchical preview folder structure - $qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')), - // Legacy flat preview folder structure - $qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')) - ))->andWhere( - $qb->expr()->eq('storage', $qb->createNamedParameter($this->rootFolder->getMountPoint()->getNumericStorageId())) - ) - ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) - ->setMaxResults(100); - - $result = $qb->executeQuery(); - $foundPreviews = $this->processQueryResult($result); - - if (!$foundPreviews) { - break; - } + $storage = $this->rootFolder->getMountPoint()->getStorage(); + if ($storage === null) { + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + return; + } - // Stop if execution time is more than one hour. - if (time() - $startTime > 3600) { - return; - } + $cache = $storage->getCache(); + $previewRootId = $cache->getId(rtrim($this->previewRootPath, '/')); + if ($previewRootId === -1) { + // No previews have ever been generated on this instance. + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + return; } - $this->appConfig->setValueBool('core', 'previewMovedDone', true); - } + $startTime = time(); - private function processQueryResult(IResult $result): bool { - $foundPreview = false; - $fileIds = []; - $flatFileIds = []; - while ($row = $result->fetch()) { - $pathSplit = explode('/', $row['path']); - assert(count($pathSplit) >= 2); - $fileId = (int)$pathSplit[count($pathSplit) - 2]; - if (count($pathSplit) === 11) { - // Hierarchical structure - if (!in_array($fileId, $fileIds)) { - $fileIds[] = $fileId; - } - } else { - // Flat structure - if (!in_array($fileId, $flatFileIds)) { - $flatFileIds[] = $fileId; + // Walk the preview folder tree via the `parent` column, which is indexed on + // every supported database platform. + // + // Depth from the preview root tells us which structure a leaf folder holds: + // - depth 1: legacy flat structure, e.g. preview//.png + // - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g//.png + $foldersToVisit = [[$previewRootId, '', 0]]; + + while ($foldersToVisit !== []) { + [$folderId, $folderName, $depth] = array_pop($foldersToVisit); + + $hasPreviewFiles = false; + foreach ($cache->getFolderContentsById($folderId) as $entry) { + if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) { + $foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1]; + } else { + $hasPreviewFiles = true; } } - $foundPreview = true; - } - foreach ($fileIds as $fileId) { - try { - $this->migrationService->migrateFileId($fileId, flatPath: false); - } catch (\Exception $e) { - $this->logger->error('Failed to migrate preview with fileId: ' . $fileId . ' (hierarchical file structure)', [ - 'exception' => $e, - ]); + if (!$hasPreviewFiles || !ctype_digit($folderName)) { + continue; } - } - foreach ($flatFileIds as $fileId) { try { - $this->migrationService->migrateFileId($fileId, flatPath: true); + $this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1); } catch (\Exception $e) { - $this->logger->error('Failed to migrate preview with fileId: ' . $fileId . ' (legacy file structure)', [ + $this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [ 'exception' => $e, ]); } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 3600) { + return; + } } - return $foundPreview; + + $this->appConfig->setValueBool('core', 'previewMovedDone', true); } } diff --git a/tests/lib/Preview/PreviewMigrationJobTest.php b/tests/lib/Preview/PreviewMigrationJobTest.php index fe9cd968799..06108c954fb 100644 --- a/tests/lib/Preview/PreviewMigrationJobTest.php +++ b/tests/lib/Preview/PreviewMigrationJobTest.php @@ -119,7 +119,6 @@ class PreviewMigrationJobTest extends TestCase { Server::get(ITimeFactory::class), $this->appConfig, $this->config, - Server::get(IDBConnection::class), Server::get(IRootFolder::class), new PreviewMigrationService( $this->config, @@ -157,7 +156,6 @@ class PreviewMigrationJobTest extends TestCase { Server::get(ITimeFactory::class), $this->appConfig, $this->config, - Server::get(IDBConnection::class), Server::get(IRootFolder::class), new PreviewMigrationService( $this->config, @@ -203,7 +201,6 @@ class PreviewMigrationJobTest extends TestCase { Server::get(ITimeFactory::class), $this->appConfig, $this->config, - Server::get(IDBConnection::class), Server::get(IRootFolder::class), new PreviewMigrationService( $this->config,