From c120a64ba2031817113a0194fd6f2337a4dc420b Mon Sep 17 00:00:00 2001 From: Julien Veyssier Date: Mon, 15 Jul 2024 13:30:59 +0200 Subject: [PATCH] feat(taskprocessing): add occ commands to list tasks and compute stats Signed-off-by: Julien Veyssier --- core/Command/TaskProcessing/ListCommand.php | 84 ++++++++++ core/Command/TaskProcessing/Statistics.php | 145 ++++++++++++++++++ .../Version30000Date20240708160048.php | 6 +- core/register_command.php | 3 + lib/composer/composer/autoload_classmap.php | 2 +- lib/composer/composer/autoload_static.php | 2 +- lib/private/TaskProcessing/Db/Task.php | 6 +- lib/private/TaskProcessing/Db/TaskMapper.php | 29 ++++ lib/private/TaskProcessing/Manager.php | 13 ++ lib/public/TaskProcessing/IManager.php | 16 ++ 10 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 core/Command/TaskProcessing/ListCommand.php create mode 100644 core/Command/TaskProcessing/Statistics.php diff --git a/core/Command/TaskProcessing/ListCommand.php b/core/Command/TaskProcessing/ListCommand.php new file mode 100644 index 00000000000..92897c8b9ea --- /dev/null +++ b/core/Command/TaskProcessing/ListCommand.php @@ -0,0 +1,84 @@ +setName('taskprocessing:task:list') + ->setDescription('list tasks') + ->addOption( + 'userIdFilter', + 'u', + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one user ID' + ) + ->addOption( + 'type', + 't', + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one task type' + ) + ->addOption( + 'customId', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one custom ID' + ) + ->addOption( + 'status', + 's', + InputOption::VALUE_OPTIONAL, + 'only get the tasks that have a specific status (STATUS_UNKNOWN=0, STATUS_SCHEDULED=1, STATUS_RUNNING=2, STATUS_SUCCESSFUL=3, STATUS_FAILED=4, STATUS_CANCELLED=5)' + ) + ->addOption( + 'scheduledAfter', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks that were scheduled after a specific date (Unix timestamp)' + ) + ->addOption( + 'endedBefore', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks that ended before a specific date (Unix timestamp)' + ); + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $userIdFilter = $input->getOption('userIdFilter'); + if ($userIdFilter === null) { + $userIdFilter = ''; + } elseif ($userIdFilter === '') { + $userIdFilter = null; + } + $type = $input->getOption('type'); + $customId = $input->getOption('customId'); + $status = $input->getOption('status'); + $scheduledAfter = $input->getOption('scheduledAfter'); + $endedBefore = $input->getOption('endedBefore'); + + $tasks = $this->taskProcessingManager->getTasks($userIdFilter, $type, $customId, $status, $scheduledAfter, $endedBefore); + $arrayTasks = array_map(fn (Task $task): array => $task->jsonSerialize(), $tasks); + + $this->writeArrayInOutputFormat($input, $output, $arrayTasks); + return 0; + } +} diff --git a/core/Command/TaskProcessing/Statistics.php b/core/Command/TaskProcessing/Statistics.php new file mode 100644 index 00000000000..b0e716337f0 --- /dev/null +++ b/core/Command/TaskProcessing/Statistics.php @@ -0,0 +1,145 @@ +setName('taskprocessing:task:stats') + ->setDescription('get statistics for tasks') + ->addOption( + 'userIdFilter', + 'u', + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one user ID' + ) + ->addOption( + 'type', + 't', + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one task type' + ) + ->addOption( + 'customId', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks for one custom ID' + ) + ->addOption( + 'status', + 's', + InputOption::VALUE_OPTIONAL, + 'only get the tasks that have a specific status (STATUS_UNKNOWN=0, STATUS_SCHEDULED=1, STATUS_RUNNING=2, STATUS_SUCCESSFUL=3, STATUS_FAILED=4, STATUS_CANCELLED=5)' + ) + ->addOption( + 'scheduledAfter', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks that were scheduled after a specific date (Unix timestamp)' + ) + ->addOption( + 'endedBefore', + null, + InputOption::VALUE_OPTIONAL, + 'only get the tasks that ended before a specific date (Unix timestamp)' + ); + parent::configure(); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + $userIdFilter = $input->getOption('userIdFilter'); + if ($userIdFilter === null) { + $userIdFilter = ''; + } elseif ($userIdFilter === '') { + $userIdFilter = null; + } + $type = $input->getOption('type'); + $customId = $input->getOption('customId'); + $status = $input->getOption('status'); + $scheduledAfter = $input->getOption('scheduledAfter'); + $endedBefore = $input->getOption('endedBefore'); + + $tasks = $this->taskProcessingManager->getTasks($userIdFilter, $type, $customId, $status, $scheduledAfter, $endedBefore); + + $stats = ['Number of tasks' => count($tasks)]; + + $maxRunningTime = 0; + $totalRunningTime = 0; + $runningTimeCount = 0; + + $maxQueuingTime = 0; + $totalQueuingTime = 0; + $queuingTimeCount = 0; + + $maxUserWaitingTime = 0; + $totalUserWaitingTime = 0; + $userWaitingTimeCount = 0; + foreach ($tasks as $task) { + // running time + if ($task->getStartedAt() !== null && $task->getEndedAt() !== null) { + $taskRunningTime = $task->getEndedAt() - $task->getStartedAt(); + $totalRunningTime += $taskRunningTime; + $runningTimeCount++; + if ($taskRunningTime >= $maxRunningTime) { + $maxRunningTime = $taskRunningTime; + } + } + // queuing time + if ($task->getScheduledAt() !== null && $task->getStartedAt() !== null) { + $taskQueuingTime = $task->getStartedAt() - $task->getScheduledAt(); + $totalQueuingTime += $taskQueuingTime; + $queuingTimeCount++; + if ($taskQueuingTime >= $maxQueuingTime) { + $maxQueuingTime = $taskQueuingTime; + } + } + // user waiting time + if ($task->getScheduledAt() !== null && $task->getEndedAt() !== null) { + $taskUserWaitingTime = $task->getEndedAt() - $task->getScheduledAt(); + $totalUserWaitingTime += $taskUserWaitingTime; + $userWaitingTimeCount++; + if ($taskUserWaitingTime >= $maxUserWaitingTime) { + $maxUserWaitingTime = $taskUserWaitingTime; + } + } + } + + if ($runningTimeCount > 0) { + $stats['Max running time'] = $maxRunningTime; + $averageRunningTime = (int)($totalRunningTime / $runningTimeCount); + $stats['Average running time'] = $averageRunningTime; + $stats['Running time count'] = $runningTimeCount; + } + if ($queuingTimeCount > 0) { + $stats['Max queuing time'] = $maxQueuingTime; + $averageQueuingTime = (int)($totalQueuingTime / $queuingTimeCount); + $stats['Average queuing time'] = $averageQueuingTime; + $stats['Queuing time count'] = $queuingTimeCount; + } + if ($userWaitingTimeCount > 0) { + $stats['Max user waiting time'] = $maxUserWaitingTime; + $averageUserWaitingTime = (int)($totalUserWaitingTime / $userWaitingTimeCount); + $stats['Average user waiting time'] = $averageUserWaitingTime; + $stats['User waiting time count'] = $userWaitingTimeCount; + } + + $this->writeArrayInOutputFormat($input, $output, $stats); + return 0; + } +} diff --git a/core/Migrations/Version30000Date20240708160048.php b/core/Migrations/Version30000Date20240708160048.php index a85eea2c974..0b5596a05a9 100644 --- a/core/Migrations/Version30000Date20240708160048.php +++ b/core/Migrations/Version30000Date20240708160048.php @@ -34,17 +34,17 @@ class Version30000Date20240708160048 extends SimpleMigrationStep { $table->addColumn('scheduled_at', Types::INTEGER, [ 'notnull' => false, - 'default' => 0, + 'default' => null, 'unsigned' => true, ]); $table->addColumn('started_at', Types::INTEGER, [ 'notnull' => false, - 'default' => 0, + 'default' => null, 'unsigned' => true, ]); $table->addColumn('ended_at', Types::INTEGER, [ 'notnull' => false, - 'default' => 0, + 'default' => null, 'unsigned' => true, ]); diff --git a/core/register_command.php b/core/register_command.php index 6560f63d797..6e89568cf9b 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -140,6 +140,9 @@ if ($config->getSystemValueBool('installed', false)) { $application->add(Server::get(Command\Security\BruteforceResetAttempts::class)); $application->add(Server::get(Command\SetupChecks::class)); $application->add(Server::get(Command\FilesMetadata\Get::class)); + + $application->add(Server::get(Command\TaskProcessing\ListCommand::class)); + $application->add(Server::get(Command\TaskProcessing\Statistics::class)); } else { $application->add(Server::get(Command\Maintenance\Install::class)); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d4c923086dc..be0073d57cb 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1327,7 +1327,7 @@ return array( 'OC\\Core\\Migrations\\Version29000Date20240131122720' => $baseDir . '/core/Migrations/Version29000Date20240131122720.php', 'OC\\Core\\Migrations\\Version30000Date20240429122720' => $baseDir . '/core/Migrations/Version30000Date20240429122720.php', 'OC\\Core\\Migrations\\Version30000Date20240708160048' => $baseDir . '/core/Migrations/Version30000Date20240708160048.php', - 'OC\\Core\\Migrations\\Version30000Date20240717111406' => $baseDir . '/core/Migrations/Version30000Date20240717111406.php', + 'OC\\Core\\Migrations\\Version30000Date20240717111406' => $baseDir . '/core/Migrations/Version30000Date20240717111406.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 34a2eb8ffe2..0be03b8f011 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1360,7 +1360,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version29000Date20240131122720' => __DIR__ . '/../../..' . '/core/Migrations/Version29000Date20240131122720.php', 'OC\\Core\\Migrations\\Version30000Date20240429122720' => __DIR__ . '/../../..' . '/core/Migrations/Version30000Date20240429122720.php', 'OC\\Core\\Migrations\\Version30000Date20240708160048' => __DIR__ . '/../../..' . '/core/Migrations/Version30000Date20240708160048.php', - 'OC\\Core\\Migrations\\Version30000Date20240717111406' => __DIR__ . '/../../..' . '/core/Migrations/Version30000Date20240717111406.php', + 'OC\\Core\\Migrations\\Version30000Date20240717111406' => __DIR__ . '/../../..' . '/core/Migrations/Version30000Date20240717111406.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', diff --git a/lib/private/TaskProcessing/Db/Task.php b/lib/private/TaskProcessing/Db/Task.php index ea95f32b252..6787c11344a 100644 --- a/lib/private/TaskProcessing/Db/Task.php +++ b/lib/private/TaskProcessing/Db/Task.php @@ -67,12 +67,12 @@ class Task extends Entity { /** * @var string[] */ - public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'custom_id', 'completion_expected_at', 'error_message', 'progress', 'webhook_uri', 'webhook_method']; + public static array $columns = ['id', 'last_updated', 'type', 'input', 'output', 'status', 'user_id', 'app_id', 'custom_id', 'completion_expected_at', 'error_message', 'progress', 'webhook_uri', 'webhook_method', 'scheduled_at', 'started_at', 'ended_at']; /** * @var string[] */ - public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'customId', 'completionExpectedAt', 'errorMessage', 'progress', 'webhookUri', 'webhookMethod']; + public static array $fields = ['id', 'lastUpdated', 'type', 'input', 'output', 'status', 'userId', 'appId', 'customId', 'completionExpectedAt', 'errorMessage', 'progress', 'webhookUri', 'webhookMethod', 'scheduledAt', 'startedAt', 'endedAt']; public function __construct() { @@ -91,7 +91,7 @@ class Task extends Entity { $this->addType('progress', 'float'); $this->addType('webhookUri', 'string'); $this->addType('webhookMethod', 'string'); - $this->addType('scheduleAt', 'integer'); + $this->addType('scheduledAt', 'integer'); $this->addType('startedAt', 'integer'); $this->addType('endedAt', 'integer'); } diff --git a/lib/private/TaskProcessing/Db/TaskMapper.php b/lib/private/TaskProcessing/Db/TaskMapper.php index da3910dcb3d..93c4f1b94cf 100644 --- a/lib/private/TaskProcessing/Db/TaskMapper.php +++ b/lib/private/TaskProcessing/Db/TaskMapper.php @@ -136,6 +136,35 @@ class TaskMapper extends QBMapper { return array_values($this->findEntities($qb)); } + public function findTasks(?string $userId, ?string $taskType = null, ?string $customId = null, ?int $status = null, ?int $scheduleAfter = null, ?int $endedBefore = null): array { + $qb = $this->db->getQueryBuilder(); + $qb->select(Task::$columns) + ->from($this->tableName); + + // empty string: no userId filter + if ($userId !== '') { + $qb->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId))); + } + if ($taskType !== null) { + $qb->andWhere($qb->expr()->eq('type', $qb->createPositionalParameter($taskType))); + } + if ($customId !== null) { + $qb->andWhere($qb->expr()->eq('custom_id', $qb->createPositionalParameter($customId))); + } + if ($status !== null) { + $qb->andWhere($qb->expr()->eq('status', $qb->createPositionalParameter($status, IQueryBuilder::PARAM_INT))); + } + if ($scheduleAfter !== null) { + $qb->andWhere($qb->expr()->isNotNull('scheduled_at')); + $qb->andWhere($qb->expr()->gt('scheduled_at', $qb->createPositionalParameter($scheduleAfter, IQueryBuilder::PARAM_INT))); + } + if ($endedBefore !== null) { + $qb->andWhere($qb->expr()->isNotNull('ended_at')); + $qb->andWhere($qb->expr()->lt('ended_at', $qb->createPositionalParameter($endedBefore, IQueryBuilder::PARAM_INT))); + } + return array_values($this->findEntities($qb)); + } + /** * @param int $timeout * @return int the number of deleted tasks diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 2ddfc3d6e51..1b2c924ef19 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -850,6 +850,19 @@ class Manager implements IManager { } } + public function getTasks( + ?string $userId, ?string $taskTypeId = null, ?string $customId = null, ?int $status = null, ?int $scheduleAfter = null, ?int $endedBefore = null + ): array { + try { + $taskEntities = $this->taskMapper->findTasks($userId, $taskTypeId, $customId, $status, $scheduleAfter, $endedBefore); + return array_map(fn ($taskEntity): Task => $taskEntity->toPublicTask(), $taskEntities); + } catch (\OCP\DB\Exception $e) { + throw new \OCP\TaskProcessing\Exception\Exception('There was a problem finding the tasks', 0, $e); + } catch (\JsonException $e) { + throw new \OCP\TaskProcessing\Exception\Exception('There was a problem parsing JSON after finding the tasks', 0, $e); + } + } + public function getUserTasksByApp(?string $userId, string $appId, ?string $customId = null): array { try { $taskEntities = $this->taskMapper->findUserTasksByApp($userId, $appId, $customId); diff --git a/lib/public/TaskProcessing/IManager.php b/lib/public/TaskProcessing/IManager.php index c68ad1afbac..33dfcdeb7c7 100644 --- a/lib/public/TaskProcessing/IManager.php +++ b/lib/public/TaskProcessing/IManager.php @@ -140,6 +140,22 @@ interface IManager { */ public function getUserTasks(?string $userId, ?string $taskTypeId = null, ?string $customId = null): array; + /** + * @param string|null $userId The user id that scheduled the task + * @param string|null $taskTypeId The task type id to filter by + * @param string|null $customId + * @param int|null $status The task status + * @param int|null $scheduleAfter Minimum schedule time filter + * @param int|null $endedBefore Maximum ending time filter + * @return list + * @throws Exception If the query failed + * @throws NotFoundException If the task could not be found + * @since 30.0.0 + */ + public function getTasks( + ?string $userId, ?string $taskTypeId = null, ?string $customId = null, ?int $status = null, ?int $scheduleAfter = null, ?int $endedBefore = null + ): array; + /** * @param string|null $userId * @param string $appId