feat: add command to scan external storages directly

the main use case of this over simply scanning through is the ability to provide a username and/or password for cases where login credentials are used

Signed-off-by: Robin Appelman <robin@icewind.nl>
pull/25109/head
Robin Appelman 4 years ago
parent abfbe67ec9
commit 42e14cc4c7
  1. 1
      apps/files_external/appinfo/info.xml
  2. 2
      apps/files_external/composer/composer/autoload_classmap.php
  3. 2
      apps/files_external/composer/composer/autoload_static.php
  4. 80
      apps/files_external/lib/Command/Notify.php
  5. 155
      apps/files_external/lib/Command/Scan.php
  6. 129
      apps/files_external/lib/Command/StorageAuthBase.php

@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
<command>OCA\Files_External\Command\Backends</command>
<command>OCA\Files_External\Command\Verify</command>
<command>OCA\Files_External\Command\Notify</command>
<command>OCA\Files_External\Command\Scan</command>
</commands>
<settings>

@ -19,6 +19,8 @@ return array(
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',

@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',

@ -30,9 +30,6 @@ declare(strict_types=1);
namespace OCA\Files_External\Command;
use Doctrine\DBAL\Exception\DriverException;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Notify\IChange;
@ -40,7 +37,6 @@ use OCP\Files\Notify\INotifyHandler;
use OCP\Files\Notify\IRenameChange;
use OCP\Files\Storage\INotifyStorage;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\IUserManager;
use Psr\Log\LoggerInterface;
@ -49,14 +45,14 @@ use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Notify extends Base {
class Notify extends StorageAuthBase {
public function __construct(
private GlobalStoragesService $globalService,
private IDBConnection $connection,
private LoggerInterface $logger,
private IUserManager $userManager
GlobalStoragesService $globalService,
IUserManager $userManager,
) {
parent::__construct();
parent::__construct($globalService, $userManager);
}
protected function configure(): void {
@ -97,71 +93,12 @@ class Notify extends Base {
parent::configure();
}
private function getUserOption(InputInterface $input): ?string {
if ($input->getOption('user')) {
return (string)$input->getOption('user');
}
return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
}
private function getPasswordOption(InputInterface $input): ?string {
if ($input->getOption('password')) {
return (string)$input->getOption('password');
}
return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
}
protected function execute(InputInterface $input, OutputInterface $output): int {
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
[$mount, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return self::FAILURE;
}
$noAuth = false;
$userOption = $this->getUserOption($input);
$passwordOption = $this->getPasswordOption($input);
// if only the user is provided, we get the user object to pass along to the auth backend
// this allows using saved user credentials
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;
try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}
if ($userOption) {
$mount->setBackendOption('user', $userOption);
}
if ($passwordOption) {
$mount->setBackendOption('password', $passwordOption);
}
try {
$backend = $mount->getBackend();
$backend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}
try {
$storage = $this->createStorage($mount);
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Login and/or password required</error>');
}
return self::FAILURE;
}
if (!$storage instanceof INotifyStorage) {
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
return self::FAILURE;
@ -189,11 +126,6 @@ class Notify extends Base {
return self::SUCCESS;
}
private function createStorage(StorageConfig $mount): IStorage {
$class = $mount->getBackend()->getStorageClass();
return new $class($mount->getBackendOptions());
}
private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void {
$parent = ltrim(dirname($path), '/');
if ($parent === '.') {

@ -0,0 +1,155 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files_External\Command;
use OC\Files\Cache\Scanner;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\IUserManager;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Scan extends StorageAuthBase {
protected float $execTime = 0;
protected int $foldersCounter = 0;
protected int $filesCounter = 0;
public function __construct(
GlobalStoragesService $globalService,
IUserManager $userManager
) {
parent::__construct($globalService, $userManager);
}
protected function configure(): void {
$this
->setName('files_external:scan')
->setDescription('Scan an external storage for changed files')
->addArgument(
'mount_id',
InputArgument::REQUIRED,
'the mount id of the mount to scan'
)->addOption(
'user',
'u',
InputOption::VALUE_REQUIRED,
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'password',
'p',
InputOption::VALUE_REQUIRED,
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
)->addOption(
'path',
'',
InputOption::VALUE_OPTIONAL,
'The path in the storage to scan',
''
);
parent::configure();
}
protected function execute(InputInterface $input, OutputInterface $output): int {
[, $storage] = $this->createStorage($input, $output);
if ($storage === null) {
return 1;
}
$path = $input->getOption('path');
$this->execTime = -microtime(true);
/** @var Scanner $scanner */
$scanner = $storage->getScanner();
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->filesCounter;
$this->abortIfInterrupted();
});
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
++$this->foldersCounter;
$this->abortIfInterrupted();
});
$scanner->scan($path);
$this->presentStats($output);
return 0;
}
/**
* @param OutputInterface $output
*/
protected function presentStats(OutputInterface $output): void {
// Stop the timer
$this->execTime += microtime(true);
$headers = [
'Folders', 'Files', 'Elapsed time'
];
$this->showSummary($headers, [], $output);
}
/**
* Shows a summary of operations
*
* @param string[] $headers
* @param string[] $rows
* @param OutputInterface $output
*/
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
$niceDate = $this->formatExecTime();
if (!$rows) {
$rows = [
$this->foldersCounter,
$this->filesCounter,
$niceDate,
];
}
$table = new Table($output);
$table
->setHeaders($headers)
->setRows([$rows]);
$table->render();
}
/**
* Formats microtime into a human readable format
*
* @return string
*/
protected function formatExecTime(): string {
$secs = round($this->execTime);
# convert seconds into HH:MM:SS form
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
}
}

@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files_External\Command;
use OC\Core\Command\Base;
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
use OCA\Files_External\Lib\StorageConfig;
use OCA\Files_External\NotFoundException;
use OCA\Files_External\Service\GlobalStoragesService;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IUserManager;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
abstract class StorageAuthBase extends Base {
public function __construct(
protected GlobalStoragesService $globalService,
protected IUserManager $userManager,
) {
parent::__construct();
}
private function getUserOption(InputInterface $input): ?string {
if ($input->getOption('user')) {
return (string)$input->getOption('user');
}
return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
}
private function getPasswordOption(InputInterface $input): ?string {
if ($input->getOption('password')) {
return (string)$input->getOption('password');
}
return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
}
/**
* @param InputInterface $input
* @param OutputInterface $output
* @return array
* @psalm-return array{0: StorageConfig, 1: IStorage}|array{0: null, 1: null}
*/
protected function createStorage(InputInterface $input, OutputInterface $output): array {
try {
/** @var StorageConfig|null $mount */
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
} catch (NotFoundException $e) {
$output->writeln('<error>Mount not found</error>');
return [null, null];
}
if (is_null($mount)) {
$output->writeln('<error>Mount not found</error>');
return [null, null];
}
$noAuth = false;
$userOption = $this->getUserOption($input);
$passwordOption = $this->getPasswordOption($input);
// if only the user is provided, we get the user object to pass along to the auth backend
// this allows using saved user credentials
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;
try {
$authBackend = $mount->getAuthMechanism();
$authBackend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}
if ($userOption) {
$mount->setBackendOption('user', $userOption);
}
if ($passwordOption) {
$mount->setBackendOption('password', $passwordOption);
}
try {
$backend = $mount->getBackend();
$backend->manipulateStorageConfig($mount, $user);
} catch (InsufficientDataForMeaningfulAnswerException $e) {
$noAuth = true;
} catch (StorageNotAvailableException $e) {
$noAuth = true;
}
try {
$class = $mount->getBackend()->getStorageClass();
/** @var IStorage $storage */
$storage = new $class($mount->getBackendOptions());
if (!$storage->test()) {
throw new \Exception();
}
return [$mount, $storage];
} catch (\Exception $e) {
$output->writeln('<error>Error while trying to create storage</error>');
if ($noAuth) {
$output->writeln('<error>Username and/or password required</error>');
}
return [null, null];
}
}
}
Loading…
Cancel
Save