open-dsi
/
nextcloud-server
mirror of https://github.com/nextcloud/server
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add utility command for object store objects

Browse Source 
Signed-off-by: Robin Appelman <robin@icewind.nl>
pull/38226/head
Robin Appelman 3 years ago
parent faf0e634db
commit ea88ec1350
No known key found for this signature in database
GPG Key ID: 42B69D8A64526EFB
8 changed files with 368 additions and 1 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      apps/files/appinfo/info.xml
  2. 4
      apps/files/composer/composer/autoload_classmap.php
  3. 4
      apps/files/composer/composer/autoload_static.php
  4. 78
      apps/files/lib/Command/Object/Delete.php
  5. 80
      apps/files/lib/Command/Object/Get.php
  6. 110
      apps/files/lib/Command/Object/ObjectUtil.php
  7. 84
      apps/files/lib/Command/Object/Put.php
  8. 6
      lib/private/Files/ObjectStore/S3ObjectTrait.php

3
apps/files/appinfo/info.xml
Unescape View File

@ -38,6 +38,9 @@
<command>OCA\Files\Command\Get</command>
<command>OCA\Files\Command\Put</command>
<command>OCA\Files\Command\Delete</command>
<command>OCA\Files\Command\Object\Delete</command>
<command>OCA\Files\Command\Object\Get</command>
<command>OCA\Files\Command\Object\Put</command>
</commands>
<activity>

4
apps/files/composer/composer/autoload_classmap.php
Unescape View File

@ -30,6 +30,10 @@ return array(
'OCA\\Files\\Command\\Delete' => $baseDir . '/../lib/Command/Delete.php',
'OCA\\Files\\Command\\DeleteOrphanedFiles' => $baseDir . '/../lib/Command/DeleteOrphanedFiles.php',
'OCA\\Files\\Command\\Get' => $baseDir . '/../lib/Command/Get.php',
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',
'OCA\\Files\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',

4
apps/files/composer/composer/autoload_static.php
Unescape View File

@ -45,6 +45,10 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Delete' => __DIR__ . '/..' . '/../lib/Command/Delete.php',
'OCA\\Files\\Command\\DeleteOrphanedFiles' => __DIR__ . '/..' . '/../lib/Command/DeleteOrphanedFiles.php',
'OCA\\Files\\Command\\Get' => __DIR__ . '/..' . '/../lib/Command/Get.php',
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',
'OCA\\Files\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',

78
apps/files/lib/Command/Object/Delete.php
Unescape View File

@ -0,0 +1,78 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
class Delete extends Command {
private ObjectUtil $objectUtils;
public function __construct(ObjectUtil $objectUtils) {
$this->objectUtils = $objectUtils;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:delete')
->setDescription('Delete an object from the object store')
->addArgument('object', InputArgument::REQUIRED, "Object to delete")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to delete the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return -1;
}
if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
$output->writeln("<error>Warning, object $object belongs to an existing file, deleting the object will lead to unexpected behavior if not replaced</error>");
$output->writeln(" Note: use <info>occ files:delete $fileId</info> to delete the file cleanly or <info>occ info:file $fileId</info> for more information about the file");
$output->writeln("");
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return -1;
}
/** @var QuestionHelper $helper */
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion("Delete $object? [y/N] ", false);
if ($helper->ask($input, $output, $question)) {
$objectStore->deleteObject($object);
}
return 0;
}
}

80
apps/files/lib/Command/Object/Get.php
Unescape View File

@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\Files\File;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Get extends Command {
private ObjectUtil $objectUtils;
public function __construct(ObjectUtil $objectUtils) {
$this->objectUtils = $objectUtils;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:get')
->setDescription('Get the contents of an object')
->addArgument('object', InputArgument::REQUIRED, "Object to get")
->addArgument('output', InputArgument::REQUIRED, "Target local file to output to, use - for STDOUT")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$outputName = $input->getArgument('output');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return 1;
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return 1;
} else {
try {
$source = $objectStore->readObject($object);
} catch (\Exception $e) {
$msg = $e->getMessage();
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
return 1;
}
$target = $outputName === '-' ? STDOUT : fopen($outputName, 'w');
if (!$target) {
$output->writeln("<error>Failed to open $outputName for writing</error>");
return 1;
}
stream_copy_to_stream($source, $target);
return 0;
}
}
}

110
apps/files/lib/Command/Object/ObjectUtil.php
Unescape View File

@ -0,0 +1,110 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
private IConfig $config;
private IDBConnection $connection;
public function __construct(IConfig $config, IDBConnection $connection) {
$this->config = $config;
$this->connection = $connection;
}
private function getObjectStoreConfig(): ?array {
$config = $this->config->getSystemValue('objectstore_multibucket');
if (is_array($config)) {
$config['multibucket'] = true;
return $config;
}
$config = $this->config->getSystemValue('objectstore');
if (is_array($config)) {
if (!isset($config['multibucket'])) {
$config['multibucket'] = false;
}
return $config;
} else {
return null;
}
}
public function getObjectStore(?string $bucket, OutputInterface $output): ?IObjectStore {
$config = $this->getObjectStoreConfig();
if (!$config) {
$output->writeln("<error>Instance is not using primary object store</error>");
return null;
}
if ($config['multibucket'] && !$bucket) {
$output->writeln("<error>--bucket option required</error> because <info>multi bucket</info> is enabled.");
return null;
}
if (!isset($config['arguments'])) {
throw new \Exception("no arguments configured for object store configuration");
}
if (!isset($config['class'])) {
throw new \Exception("no class configured for object store configuration");
}
if ($bucket) {
// s3, swift
$config['arguments']['bucket'] = $bucket;
// azure
$config['arguments']['container'] = $bucket;
}
$store = new $config['class']($config['arguments']);
if (!$store instanceof IObjectStore) {
throw new \Exception("configured object store class is not an object store implementation");
}
return $store;
}
/**
* Check if an object is referenced in the database
*/
public function objectExistsInDb(string $object): int|false {
if (str_starts_with($object, 'urn:oid:')) {
$fileId = (int)substr($object, strlen('urn:oid:'));
$query = $this->connection->getQueryBuilder();
$query->select('fileid')
->from('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
$result = $query->executeQuery();
if ($result->fetchOne() !== false) {
return $fileId;
} else {
return false;
}
} else {
return false;
}
}
}

84
apps/files/lib/Command/Object/Put.php
Unescape View File

@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2023 Robin Appelman <robin@icewind.nl>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\Files\Command\Object;
use OCP\Files\IMimeTypeDetector;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
class Put extends Command {
private ObjectUtil $objectUtils;
private IMimeTypeDetector $mimeTypeDetector;
public function __construct(ObjectUtil $objectUtils, IMimeTypeDetector $mimeTypeDetector) {
$this->objectUtils = $objectUtils;
$this->mimeTypeDetector = $mimeTypeDetector;
parent::__construct();
}
protected function configure(): void {
$this
->setName('files:object:put')
->setDescription('Write a file to the object store')
->addArgument('input', InputArgument::REQUIRED, "Source local path, use - to read from STDIN")
->addArgument('object', InputArgument::REQUIRED, "Object to write")
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket where to store the object, only required in cases where it can't be determined from the config");;
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$inputName = (string)$input->getArgument('input');
$objectStore = $this->objectUtils->getObjectStore($input->getOption("bucket"), $output);
if (!$objectStore) {
return -1;
}
if ($fileId = $this->objectUtils->objectExistsInDb($object)) {
$output->writeln("<error>Warning, object $object belongs to an existing file, overwriting the object contents can lead to unexpected behavior.</error>");
$output->writeln("You can use <info>occ files:put $inputName $fileId</info> to write to the file safely.");
$output->writeln("");
/** @var QuestionHelper $helper */
$helper = $this->getHelper('question');
$question = new ConfirmationQuestion("Write to the object anyway? [y/N] ", false);
if (!$helper->ask($input, $output, $question)) {
return -1;
}
}
$source = $inputName === '-' ? STDIN : fopen($inputName, 'r');
if (!$source) {
$output->writeln("<error>Failed to open $inputName</error>");
return 1;
}
$objectStore->writeObject($object, $source, $this->mimeTypeDetector->detectPath($inputName));
return 0;
}
}

6
lib/private/Files/ObjectStore/S3ObjectTrait.php
Unescape View File

@ -54,7 +54,7 @@ trait S3ObjectTrait {
* @since 7.0.0
*/
public function readObject($urn) {
return SeekableHttpStream::open(function ($range) use ($urn) {
$fh = SeekableHttpStream::open(function ($range) use ($urn) {
$command = $this->getConnection()->getCommand('GetObject', [
'Bucket' => $this->bucket,
'Key' => $urn,
@ -88,6 +88,10 @@ trait S3ObjectTrait {
$context = stream_context_create($opts);
return fopen($request->getUri(), 'r', false, $context);
});
if (!$fh) {
throw new \Exception("Failed to read object $urn");
}
return $fh;
}

Write Preview
Loading…
Cancel
Save