Merge pull request #27637 from nextcloud/version-expire-search

Use search to get versions list from cache for expiry

apps/files_versions/lib/Storage.php

@@ -37,8 +37,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
+
 namespace OCA\Files_Versions;
 
+use OC\Files\Search\SearchBinaryOperator;
+use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchQuery;
 use OC_User;
 use OC\Files\Filesystem;
 use OC\Files\View;
@@ -46,11 +50,16 @@ use OCA\Files_Versions\AppInfo\Application;
 use OCA\Files_Versions\Command\Expire;
 use OCA\Files_Versions\Events\CreateVersionEvent;
 use OCA\Files_Versions\Versions\IVersionManager;
+use OCP\Files\FileInfo;
+use OCP\Files\Folder;
+use OCP\Files\IRootFolder;
+use OCP\Files\Node;
 use OCP\Command\IBus;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\IMimeTypeDetector;
-use OCP\Files\IRootFolder;
 use OCP\Files\NotFoundException;
+use OCP\Files\Search\ISearchBinaryOperator;
+use OCP\Files\Search\ISearchComparison;
 use OCP\Files\StorageNotAvailableException;
 use OCP\IURLGenerator;
 use OCP\IUser;
@@ -495,38 +504,54 @@ class Storage {
 
 	/**
 	 * Expire versions that older than max version retention time
+	 *
 	 * @param string $uid
 	 */
 	public static function expireOlderThanMaxForUser($uid) {
+		/** @var IRootFolder $root */
+		$root = \OC::$server->get(IRootFolder::class);
+		try {
+			/** @var Folder $versionsRoot */
+			$versionsRoot = $root->get('/' . $uid . '/files_versions');
+		} catch (NotFoundException $e) {
+			return;
+		}
+
 		$expiration = self::getExpiration();
 		$threshold = $expiration->getMaxAgeAsTimestamp();
-		$versions = self::getAllVersions($uid);
-		if (!$threshold || empty($versions['all'])) {
+		if (!$threshold) {
 			return;
 		}
 
-		$toDelete = [];
-		foreach (array_reverse($versions['all']) as $key => $version) {
-			if ((int)$version['version'] < $threshold) {
-				$toDelete[$key] = $version;
-			} else {
-				//Versions are sorted by time - nothing mo to iterate.
-				break;
-			}
-		}
+		$allVersions = $versionsRoot->search(new SearchQuery(
+			new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_NOT, [
+				new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', FileInfo::MIMETYPE_FOLDER),
+			]),
+			0,
+			0,
+			[]
+		));
 
-		$view = new View('/' . $uid . '/files_versions');
-		if (!empty($toDelete)) {
-			foreach ($toDelete as $version) {
-				\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
-				self::deleteVersion($view, $version['path'] . '.v' . $version['version']);
-				\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'].'.v'.$version['version'], 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
+		/** @var Node[] $versions */
+		$versions = array_filter($allVersions, function (Node $info) use ($threshold) {
+			$versionsBegin = strrpos($info->getName(), '.v');
+			if ($versionsBegin === false) {
+				return false;
 			}
+			$version = (int)substr($info->getName(), $versionsBegin + 2);
+			return $version < $threshold;
+		});
+
+		foreach ($versions as $version) {
+			\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version->getInternalPath(), 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
+			$version->delete();
+			\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version->getInternalPath(), 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
 		}
 	}
 
 	/**
 	 * translate a timestamp into a string like "5 days ago"
+	 *
 	 * @param int $timestamp
 	 * @return string for example "5 days ago"
 	 */

apps/files_versions/tests/StorageTest.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\files_versions\tests;
+
+use OCA\Files_Versions\Expiration;
+use OCA\Files_Versions\Hooks;
+use OCA\Files_Versions\Storage;
+use OCP\Files\IRootFolder;
+use OCP\Files\NotFoundException;
+use Test\TestCase;
+use Test\Traits\UserTrait;
+
+/**
+ * @group DB
+ */
+class StorageTest extends TestCase {
+	use UserTrait;
+
+	private $versionsRoot;
+	private $userFolder;
+	private $expireTimestamp = 10;
+
+	protected function setUp(): void {
+		parent::setUp();
+
+		$expiration = $this->createMock(Expiration::class);
+		$expiration->method('getMaxAgeAsTimestamp')
+			->willReturnCallback(function () {
+				return $this->expireTimestamp;
+			});
+		$this->overwriteService(Expiration::class, $expiration);
+
+		Hooks::connectHooks();
+
+		$this->createUser('version_test', '');
+		$this->loginAsUser('version_test');
+		/** @var IRootFolder $root */
+		$root = \OC::$server->get(IRootFolder::class);
+		$this->userFolder = $root->getUserFolder('version_test');
+	}
+
+
+	protected function createPastFile(string $path, int $mtime) {
+		try {
+			$file = $this->userFolder->get($path);
+		} catch (NotFoundException $e) {
+			$file = $this->userFolder->newFile($path);
+		}
+		$file->putContent((string)$mtime);
+		$file->touch($mtime);
+	}
+
+	public function testExpireMaxAge() {
+		$this->userFolder->newFolder('folder1');
+		$this->userFolder->newFolder('folder1/sub1');
+		$this->userFolder->newFolder('folder2');
+
+		$this->createPastFile('file1', 100);
+		$this->createPastFile('file1', 500);
+		$this->createPastFile('file1', 900);
+
+		$this->createPastFile('folder1/file2', 100);
+		$this->createPastFile('folder1/file2', 200);
+		$this->createPastFile('folder1/file2', 300);
+
+		$this->createPastFile('folder1/sub1/file3', 400);
+		$this->createPastFile('folder1/sub1/file3', 500);
+		$this->createPastFile('folder1/sub1/file3', 600);
+
+		$this->createPastFile('folder2/file4', 100);
+		$this->createPastFile('folder2/file4', 600);
+		$this->createPastFile('folder2/file4', 800);
+
+		$this->assertCount(2, Storage::getVersions('version_test', 'file1'));
+		$this->assertCount(2, Storage::getVersions('version_test', 'folder1/file2'));
+		$this->assertCount(2, Storage::getVersions('version_test', 'folder1/sub1/file3'));
+		$this->assertCount(2, Storage::getVersions('version_test', 'folder2/file4'));
+
+		$this->expireTimestamp = 150;
+		Storage::expireOlderThanMaxForUser('version_test');
+
+		$this->assertCount(1, Storage::getVersions('version_test', 'file1'));
+		$this->assertCount(1, Storage::getVersions('version_test', 'folder1/file2'));
+		$this->assertCount(2, Storage::getVersions('version_test', 'folder1/sub1/file3'));
+		$this->assertCount(1, Storage::getVersions('version_test', 'folder2/file4'));
+
+		$this->expireTimestamp = 550;
+		Storage::expireOlderThanMaxForUser('version_test');
+
+		$this->assertCount(0, Storage::getVersions('version_test', 'file1'));
+		$this->assertCount(0, Storage::getVersions('version_test', 'folder1/file2'));
+		$this->assertCount(0, Storage::getVersions('version_test', 'folder1/sub1/file3'));
+		$this->assertCount(1, Storage::getVersions('version_test', 'folder2/file4'));
+	}
+}