open-dsi
/
nextcloud-server
mirror of https://github.com/nextcloud/server
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(taskprocessing): avoid generator cascade

Browse Source 
Signed-off-by: Julien Veyssier <julien-nc@posteo.net>
pull/54272/head
Julien Veyssier 2 months ago
parent e2c65b2493
commit aa2ca86fb3
No known key found for this signature in database
GPG Key ID: 4141FEE162030638
4 changed files with 89 additions and 80 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 61
      core/Command/TaskProcessing/Cleanup.php
  2. 46
      lib/private/TaskProcessing/Manager.php
  3. 29
      lib/private/TaskProcessing/RemoveOldTasksBackgroundJob.php
  4. 33
      tests/lib/TaskProcessing/TaskProcessingTest.php

61
core/Command/TaskProcessing/Cleanup.php
Unescape View File

@ -9,16 +9,25 @@ declare(strict_types=1);
namespace OC\Core\Command\TaskProcessing;
use OC\Core\Command\Base;
use OC\TaskProcessing\Db\TaskMapper;
use OC\TaskProcessing\Manager;
use OCP\Files\AppData\IAppDataFactory;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class Cleanup extends Base {
private \OCP\Files\IAppData $appData;
public function __construct(
protected Manager $taskProcessingManager,
private TaskMapper $taskMapper,
private LoggerInterface $logger,
IAppDataFactory $appDataFactory,
) {
parent::__construct();
$this->appData = $appDataFactory->get('core');
}
protected function configure() {
@ -37,20 +46,48 @@ class Cleanup extends Base {
protected function execute(InputInterface $input, OutputInterface $output): int {
$maxAgeSeconds = $input->getArgument('maxAgeSeconds') ?? Manager::MAX_TASK_AGE_SECONDS;
$output->writeln('<comment>Cleanup up tasks older than ' . $maxAgeSeconds . ' seconds and the related output files</comment>');
$cleanupResult = $this->taskProcessingManager->cleanupOldTasks($maxAgeSeconds);
foreach ($cleanupResult as $entry) {
if (isset($entry['task_id'], $entry['file_id'], $entry['file_name'])) {
$output->writeln("<info>\t - " . 'Deleted appData/core/TaskProcessing/' . $entry['file_name'] . ' (fileId: ' . $entry['file_id'] . ', taskId: ' . $entry['task_id'] . ')</info>');
} elseif (isset($entry['directory_name'])) {
$output->writeln("<info>\t - " . 'Deleted appData/core/' . $entry['directory_name'] . '/' . $entry['file_name'] . '</info>');
} elseif (isset($entry['deleted_task_count'])) {
$output->writeln("<comment>\t - " . 'Deleted ' . $entry['deleted_task_count'] . ' tasks from the database</comment>');
} elseif (isset($entry['deleted_task_id_list'])) {
foreach ($entry['deleted_task_id_list'] as $taskId) {
$output->writeln("<info>\t - " . 'Deleted task ' . $taskId . ' from the database</info>');
}
$taskIdsToCleanup = [];
try {
$fileCleanupGenerator = $this->taskProcessingManager->cleanupTaskProcessingTaskFiles($maxAgeSeconds);
foreach ($fileCleanupGenerator as $cleanedUpEntry) {
$output->writeln(
"<info>\t - " . 'Deleted appData/core/TaskProcessing/' . $cleanedUpEntry['file_name']
. ' (fileId: ' . $cleanedUpEntry['file_id'] . ', taskId: ' . $cleanedUpEntry['task_id'] . ')</info>'
);
}
$taskIdsToCleanup = $fileCleanupGenerator->getReturn();
} catch (\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks files', ['exception' => $e]);
$output->writeln('<warning>Failed to delete stale task processing tasks files</warning>');
}
try {
$deletedTaskCount = $this->taskMapper->deleteOlderThan($maxAgeSeconds);
foreach ($taskIdsToCleanup as $taskId) {
$output->writeln("<info>\t - " . 'Deleted task ' . $taskId . ' from the database</info>');
}
$output->writeln("<comment>\t - " . 'Deleted ' . $deletedTaskCount . ' tasks from the database</comment>');
} catch (\OCP\DB\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks', ['exception' => $e]);
$output->writeln('<warning>Failed to delete stale task processing tasks</warning>');
}
try {
$textToImageDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('text2image'), $maxAgeSeconds);
foreach ($textToImageDeletedFileNames as $entry) {
$output->writeln("<info>\t - " . 'Deleted appData/core/text2image/' . $entry . '</info>');
}
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
try {
$audioToTextDeletedFileNames = $this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('audio2text'), $maxAgeSeconds);
foreach ($audioToTextDeletedFileNames as $entry) {
$output->writeln("<info>\t - " . 'Deleted appData/core/audio2text/' . $entry . '</info>');
}
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
return 0;
}
}

46
lib/private/TaskProcessing/Manager.php
Unescape View File

@ -1490,58 +1490,18 @@ class Manager implements IManager {
return $ids;
}
/**
* @param int $ageInSeconds
* @return \Generator
*/
public function cleanupOldTasks(int $ageInSeconds = self::MAX_TASK_AGE_SECONDS): \Generator {
$taskIdsToCleanup = [];
try {
$fileCleanupGenerator = $this->cleanupTaskProcessingTaskFiles($ageInSeconds);
foreach ($fileCleanupGenerator as $cleanedUpEntry) {
yield $cleanedUpEntry;
}
$taskIdsToCleanup = $fileCleanupGenerator->getReturn();
} catch (\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks files', ['exception' => $e]);
}
try {
$deletedTaskCount = $this->taskMapper->deleteOlderThan($ageInSeconds);
yield ['deleted_task_id_list' => $taskIdsToCleanup];
yield ['deleted_task_count' => $deletedTaskCount];
} catch (\OCP\DB\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks', ['exception' => $e]);
}
try {
$textToImageDeletedFiles = $this->clearFilesOlderThan($this->appData->getFolder('text2image'), $ageInSeconds);
foreach ($textToImageDeletedFiles as $entry) {
yield $entry;
}
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
try {
$audioToTextDeletedFiles = $this->clearFilesOlderThan($this->appData->getFolder('audio2text'), $ageInSeconds);
foreach ($audioToTextDeletedFiles as $entry) {
yield $entry;
}
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
}
/**
* @param ISimpleFolder $folder
* @param int $ageInSeconds
* @return \Generator
*/
private function clearFilesOlderThan(ISimpleFolder $folder, int $ageInSeconds): \Generator {
public function clearFilesOlderThan(ISimpleFolder $folder, int $ageInSeconds = self::MAX_TASK_AGE_SECONDS): \Generator {
foreach ($folder->getDirectoryListing() as $file) {
if ($file->getMTime() < time() - $ageInSeconds) {
try {
$fileName = $file->getName();
$file->delete();
yield ['directory_name' => $folder->getName(), 'file_name' => $fileName];
yield $fileName;
} catch (NotPermittedException $e) {
$this->logger->warning('Failed to delete a stale task processing file', ['exception' => $e]);
}
@ -1558,7 +1518,7 @@ class Manager implements IManager {
* @throws \JsonException
* @throws \OCP\Files\NotFoundException
*/
private function cleanupTaskProcessingTaskFiles(int $ageInSeconds): \Generator {
public function cleanupTaskProcessingTaskFiles(int $ageInSeconds = self::MAX_TASK_AGE_SECONDS): \Generator {
$taskIdsToCleanup = [];
foreach ($this->taskMapper->getTasksToCleanup($ageInSeconds) as $task) {
$taskIdsToCleanup[] = $task->getId();

29
lib/private/TaskProcessing/RemoveOldTasksBackgroundJob.php
Unescape View File

@ -6,25 +6,52 @@
*/
namespace OC\TaskProcessing;
use OC\TaskProcessing\Db\TaskMapper;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\AppData\IAppDataFactory;
use Psr\Log\LoggerInterface;
class RemoveOldTasksBackgroundJob extends TimedJob {
private \OCP\Files\IAppData $appData;
public function __construct(
ITimeFactory $timeFactory,
private Manager $taskProcessingManager,
private TaskMapper $taskMapper,
private LoggerInterface $logger,
IAppDataFactory $appDataFactory,
) {
parent::__construct($timeFactory);
$this->setInterval(60 * 60 * 24);
// can be deferred to maintenance window
$this->setTimeSensitivity(self::TIME_INSENSITIVE);
$this->appData = $appDataFactory->get('core');
}
/**
* @inheritDoc
*/
protected function run($argument): void {
iterator_to_array($this->taskProcessingManager->cleanupOldTasks());
try {
iterator_to_array($this->taskProcessingManager->cleanupTaskProcessingTaskFiles());
} catch (\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks files', ['exception' => $e]);
}
try {
$this->taskMapper->deleteOlderThan(Manager::MAX_TASK_AGE_SECONDS);
} catch (\OCP\DB\Exception $e) {
$this->logger->warning('Failed to delete stale task processing tasks', ['exception' => $e]);
}
try {
iterator_to_array($this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('text2image')));
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
try {
iterator_to_array($this->taskProcessingManager->clearFilesOlderThan($this->appData->getFolder('audio2text')));
} catch (\OCP\Files\NotFoundException $e) {
// noop
}
}
}

33
tests/lib/TaskProcessing/TaskProcessingTest.php
Unescape View File

@ -943,6 +943,11 @@ class TaskProcessingTest extends \Test\TestCase {
$timeFactory->expects($this->any())->method('getDateTime')->willReturnCallback(fn () => $currentTime);
$timeFactory->expects($this->any())->method('getTime')->willReturnCallback(fn () => $currentTime->getTimestamp());
$this->taskMapper = new TaskMapper(
Server::get(IDBConnection::class),
$timeFactory,
);
$this->registrationContext->expects($this->any())->method('getTaskProcessingProviders')->willReturn([
new ServiceRegistration('test', SuccessfulSyncProvider::class)
]);
@ -963,34 +968,14 @@ class TaskProcessingTest extends \Test\TestCase {
$task = $this->manager->getTask($task->getId());
$taskMapper = new TaskMapper(
Server::get(IDBConnection::class),
$timeFactory,
);
$manager = new Manager(
$this->appConfig,
$this->coordinator,
$this->serverContainer,
Server::get(LoggerInterface::class),
$taskMapper,
$this->jobList,
Server::get(IEventDispatcher::class),
Server::get(IAppDataFactory::class),
Server::get(IRootFolder::class),
Server::get(\OC\TextToImage\Manager::class),
$this->userMountCache,
Server::get(IClientService::class),
Server::get(IAppManager::class),
Server::get(IUserManager::class),
Server::get(IUserSession::class),
Server::get(ICacheFactory::class),
);
$currentTime = $currentTime->add(new \DateInterval('P1Y'));
// run background job
$bgJob = new RemoveOldTasksBackgroundJob(
$timeFactory,
// use a locally defined manager to make sure the taskMapper uses the mocked timeFactory
$manager,
$this->manager,
$this->taskMapper,
Server::get(LoggerInterface::class),
Server::get(IAppDataFactory::class),
);
$bgJob->setArgument([]);
$bgJob->start($this->jobList);

Write Preview
Loading…
Cancel
Save