open-dsi
/
nextcloud-server
mirror of https://github.com/nextcloud/server
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cleanup shared cache

Browse Source
remotes/origin/comments-markallread-dav
Robin Appelman 10 years ago committed by Morris Jobke
parent b7710ab6d7
commit 1369f2c684
8 changed files with 70 additions and 547 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 490
      apps/files_sharing/lib/cache.php
  2. 25
      apps/files_sharing/lib/sharedstorage.php
  3. 81
      apps/files_sharing/lib/watcher.php
  4. 4
      apps/files_sharing/tests/testcase.php
  5. 4
      apps/files_versions/tests/versions.php
  6. 2
      lib/private/files/cache/wrapper/cachejail.php
  7. 2
      lib/private/files/cache/wrapper/cachepermissionsmask.php
  8. 9
      lib/private/files/cache/wrapper/cachewrapper.php

490
apps/files_sharing/lib/cache.php
Unescape View File

@ -32,228 +32,75 @@
namespace OC\Files\Cache;
use OC\User\NoUserException;
use OC\Files\Cache\Wrapper\CacheJail;
use OCP\Files\Cache\ICacheEntry;
use OCP\Share_Backend_Collection;
use OCP\Files\Storage\IStorage;
/**
* Metadata cache for shared files
*
* don't use this class directly if you need to get metadata, use \OC\Files\Filesystem::getFileInfo instead
*/
class Shared_Cache extends Cache {
private $storage;
private $files = array();
class Shared_Cache extends CacheJail {
/**
* @param \OC\Files\Storage\Shared $storage
* @var \OC\Files\Storage\Shared
*/
public function __construct($storage) {
parent::__construct($storage);
$this->storage = $storage;
}
private $storage;
/**
* Get the source cache of a shared file or folder
*
* @param string $target Shared target file path
* @return \OC\Files\Cache\Cache|false
* @var IStorage
*/
private function getSourceCache($target) {
if ($target === false || $target === $this->storage->getMountPoint()) {
$target = '';
}
$source = \OC_Share_Backend_File::getSource($target, $this->storage->getShare());
if (isset($source['path']) && isset($source['fileOwner'])) {
try {
\OC\Files\Filesystem::initMountPoints($source['fileOwner']);
} catch(NoUserException $e) {
\OC::$server->getLogger()->logException($e, ['app' => 'files_sharing']);
return false;
}
$mounts = \OC\Files\Filesystem::getMountByNumericId($source['storage']);
if (is_array($mounts) and !empty($mounts)) {
$fullPath = $mounts[0]->getMountPoint() . $source['path'];
list($storage, $internalPath) = \OC\Files\Filesystem::resolvePath($fullPath);
if ($storage) {
$this->files[$target] = $internalPath;
$cache = $storage->getCache();
$this->storageId = $storage->getId();
$this->numericId = $cache->getNumericStorageId();
return $cache;
}
}
}
return false;
}
public function getNumericStorageId() {
if (isset($this->numericId)) {
return $this->numericId;
} else {
return false;
}
}
private $sourceStorage;
/**
* get the stored metadata of a file or folder
*
* @param string|int $file
* @return ICacheEntry|false
* @var ICacheEntry
*/
public function get($file) {
$mimetypeLoader = \OC::$server->getMimeTypeLoader();
if (is_string($file)) {
$cache = $this->getSourceCache($file);
if ($cache) {
$data = $cache->get($this->files[$file]);
if ($data) {
$data['displayname_owner'] = \OC_User::getDisplayName($this->storage->getSharedFrom());
$data['path'] = $file;
if ($file === '') {
$data['is_share_mount_point'] = true;
}
$data['uid_owner'] = $this->storage->getOwner($file);
if (isset($data['permissions'])) {
$data['permissions'] &= $this->storage->getPermissions($file);
} else {
$data['permissions'] = $this->storage->getPermissions($file);
}
}
return $data;
}
} else {
$sourceId = $file;
// if we are at the root of the mount point we want to return the
// cache information for the source item
if (!is_int($sourceId) || $sourceId === 0) {
$sourceId = $this->storage->getSourceId();
}
$query = \OCP\DB::prepare(
'SELECT `fileid`, `storage`, `path`, `parent`, `name`, `mimetype`, `mimepart`,'
. ' `size`, `mtime`, `encrypted`, `storage_mtime`, `etag`, `permissions`'
. ' FROM `*PREFIX*filecache` WHERE `fileid` = ?');
$result = $query->execute(array($sourceId));
$data = $result->fetchRow();
$data['fileid'] = (int)$data['fileid'];
$data['mtime'] = (int)$data['mtime'];
$data['storage_mtime'] = (int)$data['storage_mtime'];
$data['encrypted'] = (bool)$data['encrypted'];
$data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']);
$data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']);
if ($data['storage_mtime'] === 0) {
$data['storage_mtime'] = $data['mtime'];
}
$data['size'] = (int)$data['size'];
$data['permissions'] = (int)$data['permissions'];
if (!is_int($file) || $file === 0) {
$data['path'] = '';
$data['name'] = basename($this->storage->getMountPoint());
$data['is_share_mount_point'] = true;
}
$data['permissions'] &= $this->storage->getPermissions('');
return $data;
}
return false;
}
private $sourceRootInfo;
/**
* get the metadata of all files stored in $folder
*
* @param string $folderId
* @return ICacheEntry[]|false
* @var \OCP\Files\Cache\ICache
*/
public function getFolderContentsById($folderId) {
$cache = $this->getSourceCache('');
if ($cache) {
$owner = $this->storage->getSharedFrom();
$parentPath = $this->getPathById($folderId);
if ($parentPath !== '') {
$parentPath .= '/';
}
$sourceFolderContent = $cache->getFolderContentsById($folderId);
foreach ($sourceFolderContent as &$c) {
$c['path'] = ltrim($parentPath . $c['name'], '/');
$c['uid_owner'] = $owner;
$c['displayname_owner'] = \OC_User::getDisplayName($owner);
$c['permissions'] = $c['permissions'] & $this->storage->getPermissions(false);
}
return $sourceFolderContent;
}
return false;
}
private $sourceCache;
/**
* store meta data for a file or folder
*
* @param string $file
* @param array $data
*
* @return int|false file id
* @param \OC\Files\Storage\Shared $storage
* @param IStorage $sourceStorage
* @param ICacheEntry $sourceRootInfo
*/
public function put($file, array $data) {
$file = ($file === false) ? '' : $file;
if ($cache = $this->getSourceCache($file)) {
return $cache->put($this->files[$file], $data);
}
return false;
public function __construct($storage, IStorage $sourceStorage, ICacheEntry $sourceRootInfo) {
$this->storage = $storage;
$this->sourceStorage = $sourceStorage;
$this->sourceRootInfo = $sourceRootInfo;
$this->sourceCache = $sourceStorage->getCache();
parent::__construct(
$this->sourceCache,
$this->sourceRootInfo->getPath()
);
}
/**
* get the file id for a file
*
* @param string $file
* @return int
*/
public function getId($file) {
if ($file === false) {
return $this->storage->getSourceId();
}
$cache = $this->getSourceCache($file);
if ($cache) {
return $cache->getId($this->files[$file]);
public function getNumericStorageId() {
if (isset($this->numericId)) {
return $this->numericId;
} else {
return false;
}
return -1;
}
/**
* check if a file is available in the cache
*
* @param string $file
* @return bool
*/
public function inCache($file) {
if ($file == '') {
return true;
protected function formatCacheEntry($entry) {
$path = $entry['path'];
$entry = parent::formatCacheEntry($entry);
$sharePermissions = $this->storage->getPermissions($path);
if (isset($entry['permissions'])) {
$entry['permissions'] &= $sharePermissions;
} else {
$entry['permissions'] = $sharePermissions;
}
return parent::inCache($file);
}
/**
* remove a file or folder from the cache
*
* @param string $file
*/
public function remove($file) {
$file = ($file === false) ? '' : $file;
if ($cache = $this->getSourceCache($file)) {
$cache->remove($this->files[$file]);
$entry['uid_owner'] = $this->storage->getOwner($path);
$entry['displayname_owner'] = \OC_User::getDisplayName($entry['uid_owner']);
if ($path === '') {
$entry['is_share_mount_point'] = true;
}
}
/**
* Get the storage id and path needed for a move
*
* @param string $path
* @return array [$storageId, $internalPath]
*/
protected function getMoveInfo($path) {
$cache = $this->getSourceCache($path);
$file = \OC_Share_Backend_File::getSource($path, $this->storage->getShare());
return [$cache->getNumericStorageId(), $file['path']];
return $entry;
}
/**
@ -262,259 +109,4 @@ class Shared_Cache extends Cache {
public function clear() {
// Not a valid action for Shared Cache
}
/**
* @param string $file
*
* @return int, Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE
*/
public function getStatus($file) {
if ($file == '') {
return self::COMPLETE;
}
if ($cache = $this->getSourceCache($file)) {
return $cache->getStatus($this->files[$file]);
}
return self::NOT_FOUND;
}
/**
* search for files matching $pattern
*
* @param string $pattern
* @return ICacheEntry[] of file data
*/
public function search($pattern) {
$pattern = trim($pattern, '%');
$normalizedPattern = $this->normalize($pattern);
$result = array();
$exploreDirs = array('');
while (count($exploreDirs) > 0) {
$dir = array_pop($exploreDirs);
$files = $this->getFolderContents($dir);
// no results?
if (!$files) {
// maybe it's a single shared file
$file = $this->get('');
if ($normalizedPattern === '' || stristr($file['name'], $normalizedPattern) !== false) {
$result[] = $file;
}
continue;
}
foreach ($files as $file) {
if ($normalizedPattern === '' || stristr($file['name'], $normalizedPattern) !== false) {
$result[] = $file;
}
if ($file['mimetype'] === 'httpd/unix-directory') {
$exploreDirs[] = ltrim($dir . '/' . $file['name'], '/');
}
}
}
return $result;
}
/**
* search for files by mimetype
*
* @param string $mimetype
* @return ICacheEntry[]
*/
public function searchByMime($mimetype) {
$mimepart = null;
if (strpos($mimetype, '/') === false) {
$mimepart = $mimetype;
$mimetype = null;
}
$result = array();
$exploreDirs = array('');
while (count($exploreDirs) > 0) {
$dir = array_pop($exploreDirs);
$files = $this->getFolderContents($dir);
// no results?
if (!$files) {
// maybe it's a single shared file
$file = $this->get('');
if (($mimepart && $file['mimepart'] === $mimepart) || ($mimetype && $file['mimetype'] === $mimetype)) {
$result[] = $file;
}
continue;
}
foreach ($files as $file) {
if ($file['mimetype'] === 'httpd/unix-directory') {
$exploreDirs[] = ltrim($dir . '/' . $file['name'], '/');
} else if (($mimepart && $file['mimepart'] === $mimepart) || ($mimetype && $file['mimetype'] === $mimetype)) {
$result[] = $file;
}
}
}
return $result;
}
/**
* Checks whether the given file has the given tag.
*
* @param \OCP\ITags $tagger
* @param array $fileData file data
* @param string $tag tag to check for
* @return boolean true if the given file has the expected tag,
* false otherwise
*/
private function hasTag($tagger, $fileData, $tag) {
$tags = $tagger->getTagsForObjects(array((int)$fileData['fileid']));
return (!empty($tags) && in_array($tag, current($tags)));
}
/**
* search for files by tag
*
* @param string|int $tag tag to search for
* @param string $userId owner of the tags
* @return ICacheEntry[] file data
*/
public function searchByTag($tag, $userId) {
// TODO: inject this
$tagger = \OC::$server->getTagManager()->load('files', null, null, $userId);
$result = array();
$exploreDirs = array('');
// check if root is tagged
$file = $this->get('');
if ($this->hasTag($tagger, $file, $tag)) {
$result[] = $file;
}
// FIXME: this is so wrong and unefficient, need to replace with actual DB queries
while (count($exploreDirs) > 0) {
$dir = array_pop($exploreDirs);
$files = $this->getFolderContents($dir);
if (!$files) {
continue;
}
foreach ($files as $file) {
if ($this->hasTag($tagger, $file, $tag)) {
$result[] = $file;
}
if ($file['mimetype'] === 'httpd/unix-directory') {
$exploreDirs[] = ltrim($dir . '/' . $file['name'], '/');
}
}
}
return $result;
}
/**
* update the folder size and the size of all parent folders
*
* @param string|boolean $path
* @param array $data (optional) meta data of the folder
*/
public function correctFolderSize($path, $data = null) {
$this->calculateFolderSize($path, $data);
if ($path !== '') {
$parent = dirname($path);
if ($parent === '.' or $parent === '/') {
$parent = '';
}
$this->correctFolderSize($parent);
} else {
// bubble up to source cache
$sourceCache = $this->getSourceCache($path);
if (isset($this->files[$path])) {
$parent = dirname($this->files[$path]);
if ($sourceCache) {
$sourceCache->correctFolderSize($parent);
}
}
}
}
/**
* get the size of a folder and set it in the cache
*
* @param string $path
* @param array $entry (optional) meta data of the folder
* @return int
*/
public function calculateFolderSize($path, $entry = null) {
$path = ($path === false) ? '' : $path;
if ($cache = $this->getSourceCache($path)) {
return $cache->calculateFolderSize($this->files[$path]);
}
return 0;
}
/**
* get all file ids on the files on the storage
*
* @return int[]
*/
public function getAll() {
$ids = \OCP\Share::getItemsSharedWith('file', \OC_Share_Backend_File::FORMAT_GET_ALL);
$folderBackend = \OCP\Share::getBackend('folder');
if ($folderBackend instanceof Share_Backend_Collection) {
foreach ($ids as $file) {
/** @var $folderBackend Share_Backend_Collection */
$children = $folderBackend->getChildren($file);
foreach ($children as $child) {
$ids[] = (int)$child['source'];
}
}
}
return $ids;
}
/**
* find a folder in the cache which has not been fully scanned
*
* If multiply incomplete folders are in the cache, the one with the highest id will be returned,
* use the one with the highest id gives the best result with the background scanner, since that is most
* likely the folder where we stopped scanning previously
*
* @return boolean the path of the folder or false when no folder matched
*/
public function getIncomplete() {
return false;
}
/**
* get the path of a file on this storage relative to the mount point by it's id
*
* @param int $id
* @param string $pathEnd (optional) used internally for recursive calls
* @return string|null
*/
public function getPathById($id, $pathEnd = '') {
// direct shares are easy
if ($id === $this->storage->getSourceId()) {
return ltrim($pathEnd, '/');
} else {
// if the item is a direct share we try and get the path of the parent and append the name of the item to it
list($parent, $name) = $this->getParentInfo($id);
if ($parent > 0) {
return $this->getPathById($parent, '/' . $name . $pathEnd);
} else {
return null;
}
}
}
/**
* @param integer $id
* @return array
*/
private function getParentInfo($id) {
$sql = 'SELECT `parent`, `name` FROM `*PREFIX*filecache` WHERE `fileid` = ?';
$query = \OCP\DB::prepare($sql);
$result = $query->execute(array($id));
if ($row = $result->fetchRow()) {
return array((int)$row['parent'], $row['name']);
} else {
return array(-1, '');
}
}
}

25
apps/files_sharing/lib/sharedstorage.php
Unescape View File

@ -32,6 +32,8 @@ namespace OC\Files\Storage;
use OC\Files\Filesystem;
use OCA\Files_Sharing\ISharedStorage;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\Storage\IStorage;
use OCP\Lock\ILockingProvider;
/**
@ -41,7 +43,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage {
private $share; // the shared resource
private $files = array();
private static $isInitialized = array();
/**
* @var \OC\Files\View
@ -55,6 +56,16 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage {
private $initialized = false;
/**
* @var ICacheEntry
*/
private $sourceRootInfo;
/**
* @var IStorage
*/
private $sourceStorage;
public function __construct($arguments) {
$this->share = $arguments['share'];
$this->ownerView = $arguments['ownerView'];
@ -67,6 +78,9 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage {
}
$this->initialized = true;
Filesystem::initMountPoints($this->share['uid_owner']);
$sourcePath = $this->ownerView->getPath($this->share['file_source']);
list($this->sourceStorage, $sourceInternalPath) = $this->ownerView->resolvePath($sourcePath);
$this->sourceRootInfo = $this->sourceStorage->getCache()->get($sourceInternalPath);
}
/**
@ -543,7 +557,7 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage {
if (!$storage) {
$storage = $this;
}
return new \OC\Files\Cache\Shared_Cache($storage);
return new \OC\Files\Cache\Shared_Cache($storage, $this->sourceStorage, $this->sourceRootInfo);
}
public function getScanner($path = '', $storage = null) {
@ -553,13 +567,6 @@ class Shared extends \OC\Files\Storage\Common implements ISharedStorage {
return new \OC\Files\Cache\SharedScanner($storage);
}
public function getWatcher($path = '', $storage = null) {
if (!$storage) {
$storage = $this;
}
return new \OC\Files\Cache\Shared_Watcher($storage);
}
public function getPropagator($storage = null) {
if (!$storage) {
$storage = $this;

81
apps/files_sharing/lib/watcher.php
Unescape View File

@ -1,81 +0,0 @@
<?php
/**
* @author Christopher Schäpers <kondou@ts.unde.re>
* @author Michael Gapczynski <GapczynskiM@gmail.com>
* @author Morris Jobke <hey@morrisjobke.de>
* @author Robin Appelman <icewind@owncloud.com>
* @author Vincent Petry <pvince81@owncloud.com>
*
* @copyright Copyright (c) 2016, ownCloud, Inc.
* @license AGPL-3.0
*
* This code is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License, version 3,
* along with this program. If not, see <http://www.gnu.org/licenses/>
*
*/
namespace OC\Files\Cache;
use OCP\Files\Cache\ICacheEntry;
/**
* check the storage backends for updates and change the cache accordingly
*/
class Shared_Watcher extends Watcher {
/**
* @var \OC\Files\Storage\Shared $storage
*/
protected $storage;
/**
* Update the cache for changes to $path
*
* @param string $path
* @param ICacheEntry $cachedData
*/
public function update($path, $cachedData) {
parent::update($path, $cachedData);
// since parent::update() has already updated the size of the subdirs,
// only apply the update to the owner's parent dirs
// find last parent before reaching the shared storage root,
// which is the actual shared dir from the owner
$sepPos = strpos($path, '/');
if ($sepPos > 0) {
$baseDir = substr($path, 0, $sepPos);
} else {
$baseDir = $path;
}
// find the path relative to the data dir
$file = $this->storage->getFile($baseDir);
$view = new \OC\Files\View('/' . $file['fileOwner']);
// find the owner's storage and path
/** @var \OC\Files\Storage\Storage $storage */
list($storage, $internalPath) = $view->resolvePath($file['path']);
// update the parent dirs' sizes in the owner's cache
$storage->getCache()->correctFolderSize(dirname($internalPath));
}
/**
* remove deleted files in $path from the cache
*
* @param string $path
*/
public function cleanFolder($path) {
if ($path != '') {
parent::cleanFolder($path);
}
}
}

4
apps/files_sharing/tests/testcase.php
Unescape View File

@ -174,9 +174,9 @@ abstract class TestCase extends \Test\TestCase {
*/
protected static function resetStorage() {
$storage = new \ReflectionClass('\OC\Files\Storage\Shared');
$isInitialized = $storage->getProperty('isInitialized');
$isInitialized = $storage->getProperty('initialized');
$isInitialized->setAccessible(true);
$isInitialized->setValue(array());
$isInitialized->setValue($storage, false);
$isInitialized->setAccessible(false);
}

4
apps/files_versions/tests/versions.php
Unescape View File

@ -802,9 +802,9 @@ class Test_Files_Versioning extends \Test\TestCase {
}
$storage = new \ReflectionClass('\OC\Files\Storage\Shared');
$isInitialized = $storage->getProperty('isInitialized');
$isInitialized = $storage->getProperty('initialized');
$isInitialized->setAccessible(true);
$isInitialized->setValue(array());
$isInitialized->setValue($storage, false);
$isInitialized->setAccessible(false);
\OC_Util::tearDownFS();

2
lib/private/files/cache/wrapper/cachejail.php vendored
Unescape View File

@ -34,7 +34,7 @@ class CacheJail extends CacheWrapper {
protected $root;
/**
* @param \OC\Files\Cache\Cache $cache
* @param \OCP\Files\Cache\ICache $cache
* @param string $root
*/
public function __construct($cache, $root) {

2
lib/private/files/cache/wrapper/cachepermissionsmask.php vendored
Unescape View File

@ -29,7 +29,7 @@ class CachePermissionsMask extends CacheWrapper {
protected $mask;
/**
* @param \OC\Files\Cache\Cache $cache
* @param \OCP\Files\Cache\ICache $cache
* @param int $mask
*/
public function __construct($cache, $mask) {

9
lib/private/files/cache/wrapper/cachewrapper.php vendored
Unescape View File

@ -26,15 +26,16 @@ namespace OC\Files\Cache\Wrapper;
use OC\Files\Cache\Cache;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\Cache\ICache;
class CacheWrapper extends Cache {
/**
* @var \OC\Files\Cache\Cache
* @var \OCP\Files\Cache\ICache
*/
protected $cache;
/**
* @param \OC\Files\Cache\Cache $cache
* @param \OCP\Files\Cache\ICache $cache
*/
public function __construct($cache) {
$this->cache = $cache;
@ -159,6 +160,10 @@ class CacheWrapper extends Cache {
$this->cache->move($source, $target);
}
public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
$this->cache->moveFromCache($sourceCache, $sourcePath, $targetPath);
}
/**
* remove all entries for files that are stored on the storage from the cache
*/

Write Preview
Loading…
Cancel
Save