open-dsi
/
nextcloud-server
mirror of https://github.com/nextcloud/server
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make Cache\Updater per storage

Browse Source
remotes/origin/fix-delete-homeidr-on-userdelete
Robin Appelman 10 years ago
parent 50f6817ce9
commit b025f07fb7
9 changed files with 161 additions and 150 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      apps/dav/lib/connector/sabre/file.php
  2. 2
      apps/encryption/lib/migration.php
  3. 6
      apps/files_trashbin/lib/trashbin.php
  4. 168
      lib/private/files/cache/updater.php
  5. 12
      lib/private/files/storage/common.php
  6. 8
      lib/private/files/storage/storage.php
  7. 7
      lib/private/files/storage/wrapper/wrapper.php
  8. 63
      lib/private/files/view.php
  9. 41
      tests/lib/files/cache/updater.php

4
apps/dav/lib/connector/sabre/file.php
Unescape View File

@ -192,7 +192,7 @@ class File extends Node implements IFile {
}
// since we skipped the view we need to scan and emit the hooks ourselves
$this->fileView->getUpdater()->update($this->path);
$storage->getUpdater()->update($internalPath);
if ($view) {
$this->emitPostHooks($exists);
@ -438,7 +438,7 @@ class File extends Node implements IFile {
$this->fileView->changeLock($targetPath, ILockingProvider::LOCK_SHARED);
// since we skipped the view we need to scan and emit the hooks ourselves
$this->fileView->getUpdater()->update($targetPath);
$targetStorage->getUpdater()->update($targetInternalPath);
$this->emitPostHooks($exists, $targetPath);

2
apps/encryption/lib/migration.php
Unescape View File

@ -50,7 +50,7 @@ class Migration {
*/
public function __construct(IConfig $config, View $view, IDBConnection $connection, ILogger $logger) {
$this->view = $view;
$this->view->getUpdater()->disable();
$this->view->disableCacheUpdate();
$this->connection = $connection;
$this->moduleId = \OCA\Encryption\Crypto\Encryption::ID;
$this->config = $config;

6
apps/files_trashbin/lib/trashbin.php
Unescape View File

@ -235,7 +235,7 @@ class Trashbin {
return false;
}
$ownerView->getUpdater()->rename('/files/' . $ownerPath, $trashPath);
$trashStorage->getUpdater()->renameFromStorage($sourceStorage, $sourceInternalPath, $trashInternalPath);
if ($sizeOfAddedFiles !== false) {
$size = $sizeOfAddedFiles;
@ -323,7 +323,7 @@ class Trashbin {
$result = $targetStorage->moveFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
if ($result) {
$view->getUpdater()->rename($source, $target);
$targetStorage->getUpdater()->renameFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
}
return $result;
}
@ -345,7 +345,7 @@ class Trashbin {
$result = $targetStorage->copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
if ($result) {
$view->getUpdater()->update($target);
$targetStorage->getUpdater()->update($targetInternalPath);
}
return $result;
}

168
lib/private/files/cache/updater.php vendored
Unescape View File

@ -29,8 +29,6 @@ namespace OC\Files\Cache;
/**
* Update the cache and propagate changes
*
* Unlike most other classes an Updater is not related to a specific storage but handles updates for all storages in a users filesystem.
* This is needed because the propagation of mtime and etags need to cross storage boundaries
*/
class Updater {
/**
@ -39,21 +37,33 @@ class Updater {
protected $enabled = true;
/**
* @var \OC\Files\View
* @var \OC\Files\Storage\Storage
*/
protected $view;
protected $storage;
/**
* @var \OC\Files\Cache\ChangePropagator
* @var \OC\Files\Cache\Propagator
*/
protected $propagator;
/**
* @param \OC\Files\View $view the view the updater works on, usually the view of the logged in user
* @var Scanner
*/
public function __construct($view) {
$this->view = $view;
$this->propagator = new ChangePropagator($view);
protected $scanner;
/**
* @var Cache
*/
protected $cache;
/**
* @param \OC\Files\Storage\Storage $storage
*/
public function __construct(\OC\Files\Storage\Storage $storage) {
$this->storage = $storage;
$this->propagator = $storage->getPropagator();
$this->scanner = $storage->getScanner();
$this->cache = $storage->getCache();
}
/**
@ -73,7 +83,7 @@ class Updater {
/**
* Get the propagator for etags and mtime for the view the updater works on
*
* @return ChangePropagator
* @return Propagator
*/
public function getPropagator() {
return $this->propagator;
@ -89,8 +99,7 @@ class Updater {
if (Scanner::isPartialFile($path)) {
return;
}
$this->propagator->addChange($path);
$this->propagator->propagateChanges($time);
$this->propagator->propagateChange($path, $time);
}
/**
@ -103,20 +112,14 @@ class Updater {
if (!$this->enabled or Scanner::isPartialFile($path)) {
return;
}
/**
* @var \OC\Files\Storage\Storage $storage
* @var string $internalPath
*/
list($storage, $internalPath) = $this->view->resolvePath($path);
if ($storage) {
$this->propagator->addChange($path);
$cache = $storage->getCache($internalPath);
$scanner = $storage->getScanner($internalPath);
$data = $scanner->scan($internalPath, Scanner::SCAN_SHALLOW, -1, false);
$this->correctParentStorageMtime($storage, $internalPath);
$cache->correctFolderSize($internalPath, $data);
$this->propagator->propagateChanges($time);
if (is_null($time)) {
$time = time();
}
$data = $this->scanner->scan($path, Scanner::SCAN_SHALLOW, -1, false);
$this->correctParentStorageMtime($path);
$this->cache->correctFolderSize($path, $data);
$this->propagator->propagateChange($path, $time);
}
/**
@ -128,87 +131,71 @@ class Updater {
if (!$this->enabled or Scanner::isPartialFile($path)) {
return;
}
/**
* @var \OC\Files\Storage\Storage $storage
* @var string $internalPath
*/
list($storage, $internalPath) = $this->view->resolvePath($path);
if ($storage) {
$parent = dirname($internalPath);
if ($parent === '.') {
$parent = '';
}
$this->propagator->addChange($path);
$cache = $storage->getCache($internalPath);
$cache->remove($internalPath);
$cache->correctFolderSize($parent);
$this->correctParentStorageMtime($storage, $internalPath);
$this->propagator->propagateChanges();
$parent = dirname($path);
if ($parent === '.') {
$parent = '';
}
$this->cache->remove($path);
$this->cache->correctFolderSize($parent);
$this->correctParentStorageMtime($path);
$this->propagator->propagateChange($path, time());
}
/**
* Rename a file or folder in the cache and update the size, etag and mtime of the parent folders
*
* @param \OC\Files\Storage\Storage $sourceStorage
* @param string $source
* @param string $target
*/
public function rename($source, $target) {
public function renameFromStorage(\OC\Files\Storage\Storage $sourceStorage, $source, $target) {
if (!$this->enabled or Scanner::isPartialFile($source) or Scanner::isPartialFile($target)) {
return;
}
/**
* @var \OC\Files\Storage\Storage $sourceStorage
* @var \OC\Files\Storage\Storage $targetStorage
* @var string $sourceInternalPath
* @var string $targetInternalPath
*/
list($sourceStorage, $sourceInternalPath) = $this->view->resolvePath($source);
// if it's a moved mountpoint we dont need to do anything
if ($sourceInternalPath === '') {
return;
}
list($targetStorage, $targetInternalPath) = $this->view->resolvePath($target);
if ($sourceStorage && $targetStorage) {
$targetCache = $targetStorage->getCache($sourceInternalPath);
if ($sourceStorage->getCache($sourceInternalPath)->inCache($sourceInternalPath)) {
if ($targetCache->inCache($targetInternalPath)) {
$targetCache->remove($targetInternalPath);
}
if ($sourceStorage === $targetStorage) {
$targetCache->move($sourceInternalPath, $targetInternalPath);
} else {
$targetCache->moveFromCache($sourceStorage->getCache(), $sourceInternalPath, $targetInternalPath);
}
$time = time();
$sourceCache = $sourceStorage->getCache($source);
$sourceUpdater = $sourceStorage->getUpdater();
$sourcePropagator = $sourceStorage->getPropagator();
if ($sourceCache->inCache($source)) {
if ($this->cache->inCache($target)) {
$this->cache->remove($target);
}
if (pathinfo($sourceInternalPath, PATHINFO_EXTENSION) !== pathinfo($targetInternalPath, PATHINFO_EXTENSION)) {
// handle mime type change
$mimeType = $targetStorage->getMimeType($targetInternalPath);
$fileId = $targetCache->getId($targetInternalPath);
$targetCache->update($fileId, array('mimetype' => $mimeType));
if ($sourceStorage === $this->storage) {
$this->cache->move($source, $target);
} else {
$this->cache->moveFromCache($sourceCache, $source, $target);
}
}
$targetCache->correctFolderSize($sourceInternalPath);
$targetCache->correctFolderSize($targetInternalPath);
$this->correctParentStorageMtime($sourceStorage, $sourceInternalPath);
$this->correctParentStorageMtime($targetStorage, $targetInternalPath);
$this->updateStorageMTimeOnly($targetStorage, $targetInternalPath);
$this->propagator->addChange($source);
$this->propagator->addChange($target);
$this->propagator->propagateChanges();
if (pathinfo($source, PATHINFO_EXTENSION) !== pathinfo($target, PATHINFO_EXTENSION)) {
// handle mime type change
$mimeType = $this->storage->getMimeType($target);
$fileId = $this->cache->getId($target);
$this->cache->update($fileId, ['mimetype' => $mimeType]);
}
$sourceCache->correctFolderSize($source);
$this->cache->correctFolderSize($target);
$sourceUpdater->correctParentStorageMtime($source);
$this->correctParentStorageMtime($target);
$this->updateStorageMTimeOnly($target);
$sourcePropagator->propagateChange($source, $time);
$this->propagator->propagateChange($target, $time);
}
private function updateStorageMTimeOnly($storage, $internalPath) {
$cache = $storage->getCache();
$fileId = $cache->getId($internalPath);
private function updateStorageMTimeOnly($internalPath) {
$fileId = $this->cache->getId($internalPath);
if ($fileId !== -1) {
$cache->update(
$this->cache->update(
$fileId, [
'mtime' => null, // this magic tells it to not overwrite mtime
'storage_mtime' => $storage->filemtime($internalPath)
'storage_mtime' => $this->storage->filemtime($internalPath)
]
);
}
@ -217,20 +204,13 @@ class Updater {
/**
* update the storage_mtime of the direct parent in the cache to the mtime from the storage
*
* @param \OC\Files\Storage\Storage $storage
* @param string $internalPath
*/
private function correctParentStorageMtime($storage, $internalPath) {
$cache = $storage->getCache();
$parentId = $cache->getParentId($internalPath);
public function correctParentStorageMtime($internalPath) {
$parentId = $this->cache->getParentId($internalPath);
$parent = dirname($internalPath);
if ($parentId != -1) {
$cache->update($parentId, array('storage_mtime' => $storage->filemtime($parent)));
$this->cache->update($parentId, array('storage_mtime' => $this->storage->filemtime($parent)));
}
}
public function __destruct() {
// propagate any leftover changes
$this->propagator->propagateChanges();
}
}

12
lib/private/files/storage/common.php
Unescape View File

@ -39,6 +39,7 @@ namespace OC\Files\Storage;
use OC\Files\Cache\Cache;
use OC\Files\Cache\Propagator;
use OC\Files\Cache\Scanner;
use OC\Files\Cache\Updater;
use OC\Files\Filesystem;
use OC\Files\Cache\Watcher;
use OCP\Files\FileNameTooLongException;
@ -67,6 +68,7 @@ abstract class Common implements Storage {
protected $watcher;
protected $propagator;
protected $storageCache;
protected $updater;
protected $mountOptions = [];
@ -363,6 +365,16 @@ abstract class Common implements Storage {
return $this->propagator;
}
public function getUpdater($storage = null) {
if (!$storage) {
$storage = $this;
}
if (!isset($this->updater)) {
$this->updater = new Updater($storage);
}
return $this->updater;
}
public function getStorageCache($storage = null) {
if (!$storage) {
$storage = $this;

8
lib/private/files/storage/storage.php
Unescape View File

@ -75,6 +75,14 @@ interface Storage extends \OCP\Files\Storage {
*/
public function getPropagator($storage = null);
/**
* get a updater instance for the cache
*
* @param \OC\Files\Storage\Storage (optional) the storage to pass to the watcher
* @return \OC\Files\Cache\Updater
*/
public function getUpdater($storage = null);
/**
* @return \OC\Files\Cache\Storage
*/

7
lib/private/files/storage/wrapper/wrapper.php
Unescape View File

@ -437,6 +437,13 @@ class Wrapper implements \OC\Files\Storage\Storage {
return $this->storage->getPropagator($storage);
}
public function getUpdater($storage = null) {
if (!$storage) {
$storage = $this;
}
return $this->storage->getUpdater($storage);
}
/**
* @return \OC\Files\Cache\Storage
*/

63
lib/private/files/view.php
Unescape View File

@ -46,6 +46,7 @@ namespace OC\Files;
use Icewind\Streams\CallbackWrapper;
use OC\Files\Cache\Updater;
use OC\Files\Mount\MoveableMount;
use OC\Files\Storage\Storage;
use OC\User\User;
use OCP\Files\FileNameTooLongException;
use OCP\Files\InvalidCharacterInPathException;
@ -76,9 +77,6 @@ class View {
/** @var string */
private $fakeRoot = '';
/** @var \OC\Files\Cache\Updater */
protected $updater;
/**
* @var \OCP\Lock\ILockingProvider
*/
@ -86,6 +84,8 @@ class View {
private $lockingEnabled;
private $updaterEnabled = true;
/**
* @param string $root
* @throws \Exception If $root contains an invalid path
@ -99,7 +99,6 @@ class View {
}
$this->fakeRoot = $root;
$this->updater = new Updater($this);
$this->lockingProvider = \OC::$server->getLockingProvider();
$this->lockingEnabled = !($this->lockingProvider instanceof \OC\Lock\NoopLockingProvider);
}
@ -286,6 +285,35 @@ class View {
}
}
public function disableCacheUpdate() {
$this->updaterEnabled = false;
}
public function enableCacheUpdate() {
$this->updaterEnabled = true;
}
protected function writeUpdate(Storage $storage, $internalPath, $time = null) {
if ($this->updaterEnabled) {
if (is_null($time)) {
$time = time();
}
$storage->getUpdater()->update($internalPath, $time);
}
}
protected function removeUpdate(Storage $storage, $internalPath) {
if ($this->updaterEnabled) {
$storage->getUpdater()->remove($internalPath);
}
}
protected function renameUpdate(Storage $sourceStorage, Storage $targetStorage, $sourceInternalPath, $targetInternalPath) {
if ($this->updaterEnabled) {
$targetStorage->getUpdater()->renameFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
}
}
/**
* @param string $path
* @return bool|mixed
@ -569,7 +597,7 @@ class View {
fclose($target);
fclose($data);
$this->updater->update($path);
$this->writeUpdate($storage, $internalPath);
$this->changeLock($path, ILockingProvider::LOCK_SHARED);
@ -703,10 +731,11 @@ class View {
if ((Cache\Scanner::isPartialFile($path1) && !Cache\Scanner::isPartialFile($path2)) && $result !== false) {
// if it was a rename from a part file to a regular file it was a write and not a rename operation
$this->updater->update($path2);
$this->writeUpdate($storage2, $internalPath2);
} else if ($result) {
if ($internalPath1 !== '') { // dont do a cache update for moved mounts
$this->updater->rename($path1, $path2);
$this->renameUpdate($storage1, $storage2, $internalPath1, $internalPath2);
}
}
@ -803,7 +832,7 @@ class View {
$result = $storage2->copyFromStorage($storage1, $internalPath1, $internalPath2);
}
$this->updater->update($path2);
$this->writeUpdate($storage2, $internalPath2);
$this->changeLock($path2, ILockingProvider::LOCK_SHARED);
$lockTypePath2 = ILockingProvider::LOCK_SHARED;
@ -1013,6 +1042,7 @@ class View {
}
$run = $this->runHooks($hooks, $path);
/** @var \OC\Files\Storage\Storage $storage */
list($storage, $internalPath) = Filesystem::resolvePath($absolutePath . $postFix);
if ($run and $storage) {
if (in_array('write', $hooks) || in_array('delete', $hooks)) {
@ -1034,13 +1064,13 @@ class View {
}
if (in_array('delete', $hooks) and $result) {
$this->updater->remove($path);
$this->removeUpdate($storage, $internalPath);
}
if (in_array('write', $hooks) and $operation !== 'fopen') {
$this->updater->update($path);
$this->writeUpdate($storage, $internalPath);
}
if (in_array('touch', $hooks)) {
$this->updater->update($path, $extraParam);
$this->writeUpdate($storage, $internalPath, $extraParam);
}
if ((in_array('write', $hooks) || in_array('delete', $hooks)) && ($operation !== 'fopen' || $result === false)) {
@ -1205,7 +1235,7 @@ class View {
} else if (!Cache\Scanner::isPartialFile($internalPath) && $watcher->needsUpdate($internalPath, $data)) {
$this->lockFile($relativePath, ILockingProvider::LOCK_SHARED);
$watcher->update($internalPath, $data);
$this->updater->propagate($relativePath);
$storage->getPropagator()->propagateChange($internalPath, time());
$data = $cache->get($internalPath);
$this->unlockFile($relativePath, ILockingProvider::LOCK_SHARED);
}
@ -1242,7 +1272,7 @@ class View {
if ($storage) {
$data = $this->getCacheEntry($storage, $internalPath, $relativePath);
if(!is_array($data)) {
if (!is_array($data)) {
return false;
}
@ -1691,13 +1721,6 @@ class View {
);
}
/**
* @return Updater
*/
public function getUpdater() {
return $this->updater;
}
/**
* @param string $path
* @param string $fileName

41
tests/lib/files/cache/updater.php vendored
Unescape View File

@ -39,9 +39,7 @@ class Updater extends \Test\TestCase {
$this->loginAsUser();
$this->storage = new Temporary(array());
Filesystem::mount($this->storage, array(), '/');
$this->view = new View('');
$this->updater = new \OC\Files\Cache\Updater($this->view);
$this->updater = $this->storage->getUpdater();
$this->cache = $this->storage->getCache();
}
@ -56,7 +54,7 @@ class Updater extends \Test\TestCase {
$this->storage->file_put_contents('foo.txt', 'bar');
$this->assertFalse($this->cache->inCache('foo.txt'));
$this->updater->update('/foo.txt');
$this->updater->update('foo.txt');
$this->assertTrue($this->cache->inCache('foo.txt'));
$cached = $this->cache->get('foo.txt');
@ -65,7 +63,9 @@ class Updater extends \Test\TestCase {
}
public function testUpdatedFile() {
$this->view->file_put_contents('/foo.txt', 'bar');
$this->storage->file_put_contents('foo.txt', 'bar');
$this->updater->update('foo.txt');
$cached = $this->cache->get('foo.txt');
$this->assertEquals(3, $cached['size']);
$this->assertEquals('text/plain', $cached['mimetype']);
@ -131,7 +131,7 @@ class Updater extends \Test\TestCase {
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->assertFalse($this->cache->inCache('bar.txt'));
$this->updater->rename('foo.txt', 'bar.txt');
$this->updater->renameFromStorage($this->storage, 'foo.txt', 'bar.txt');
$this->assertFalse($this->cache->inCache('foo.txt'));
$this->assertTrue($this->cache->inCache('bar.txt'));
@ -149,7 +149,7 @@ class Updater extends \Test\TestCase {
$cached = $this->cache->get('bar.txt');
$this->updater->rename('foo.txt', 'bar.txt');
$this->updater->renameFromStorage($this->storage, 'foo.txt', 'bar.txt');
$this->assertFalse($this->cache->inCache('foo.txt'));
$this->assertTrue($this->cache->inCache('bar.txt'));
@ -186,7 +186,7 @@ class Updater extends \Test\TestCase {
// some storages (like Dropbox) change storage mtime on rename
$this->storage->touch('sub2/bar.txt', $testmtime);
$this->updater->rename('sub/foo.txt', 'sub2/bar.txt');
$this->updater->renameFromStorage($this->storage, 'sub/foo.txt', 'sub2/bar.txt');
$cachedTargetParent = $this->cache->get('sub2');
$cachedTarget = $this->cache->get('sub2/bar.txt');
@ -212,26 +212,6 @@ class Updater extends \Test\TestCase {
$this->assertFalse($this->cache->inCache('foo.txt'));
}
public function testMoveDisabled() {
$this->storage->file_put_contents('foo.txt', 'qwerty');
$this->updater->update('foo.txt');
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->assertFalse($this->cache->inCache('bar.txt'));
$cached = $this->cache->get('foo.txt');
$this->storage->rename('foo.txt', 'bar.txt');
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->assertFalse($this->cache->inCache('bar.txt'));
$this->updater->disable();
$this->updater->rename('foo.txt', 'bar.txt');
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->assertFalse($this->cache->inCache('bar.txt'));
}
public function testMoveCrossStorage() {
$storage2 = new Temporary(array());
$cache2 = $storage2->getCache();
@ -251,7 +231,7 @@ class Updater extends \Test\TestCase {
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->assertFalse($cache2->inCache('bar.txt'));
$this->updater->rename('foo.txt', 'bar/bar.txt');
$storage2->getUpdater()->renameFromStorage($this->storage, 'foo.txt', 'bar.txt');
$this->assertFalse($this->cache->inCache('foo.txt'));
$this->assertTrue($cache2->inCache('bar.txt'));
@ -288,7 +268,8 @@ class Updater extends \Test\TestCase {
$cached[] = $this->cache->get('foo/bar/bar.txt');
// add extension to trigger the possible mimetype change
$this->view->rename('/foo', '/bar/foo.b');
$storage2->moveFromStorage($this->storage, 'foo', 'foo.b');
$storage2->getUpdater()->renameFromStorage($this->storage, 'foo', 'foo.b');
$this->assertFalse($this->cache->inCache('foo'));
$this->assertFalse($this->cache->inCache('foo/foo.txt'));

Write Preview
Loading…
Cancel
Save