From 33f26c8da236761a41b874421ba07979ab1d5bbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Julius=20H=C3=A4rtl?= <jus@bitgrid.net>
Date: Wed, 14 Feb 2018 15:04:08 +0100
Subject: [PATCH] Frontend regression testing with puppeteer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Julius Härtl <jus@bitgrid.net>
---
 .drone.yml                               |  48 +++++
 .gitignore                               |   3 +
 tests/ui-regression/config.js            |  57 ++++++
 tests/ui-regression/helper.js            | 207 +++++++++++++++++++++
 tests/ui-regression/out/index.html       | 219 +++++++++++++++++++++++
 tests/ui-regression/package.json         |  21 +++
 tests/ui-regression/runTests.js          | 129 +++++++++++++
 tests/ui-regression/test/appsSpec.js     |  60 +++++++
 tests/ui-regression/test/filesSpec.js    | 101 +++++++++++
 tests/ui-regression/test/installSpec.js  |  75 ++++++++
 tests/ui-regression/test/loginSpec.js    |  75 ++++++++
 tests/ui-regression/test/publicSpec.js   | 102 +++++++++++
 tests/ui-regression/test/settingsSpec.js |  76 ++++++++
 13 files changed, 1173 insertions(+)
 create mode 100644 tests/ui-regression/config.js
 create mode 100644 tests/ui-regression/helper.js
 create mode 100644 tests/ui-regression/out/index.html
 create mode 100644 tests/ui-regression/package.json
 create mode 100644 tests/ui-regression/runTests.js
 create mode 100644 tests/ui-regression/test/appsSpec.js
 create mode 100644 tests/ui-regression/test/filesSpec.js
 create mode 100644 tests/ui-regression/test/installSpec.js
 create mode 100644 tests/ui-regression/test/loginSpec.js
 create mode 100644 tests/ui-regression/test/publicSpec.js
 create mode 100644 tests/ui-regression/test/settingsSpec.js

diff --git a/.drone.yml b/.drone.yml
index 709079bac01..2bcc1e281ab 100644
--- a/.drone.yml
+++ b/.drone.yml
@@ -703,6 +703,32 @@ pipeline:
     when:
       matrix:
         TEST: memcache-redis-cluster
+  ui-regression:
+    image: nextcloudci/ui-regression:ui-regression-1
+    commands:
+      - cd tests/ui-regression
+      - npm install
+      - chown -R pptruser out node_modules
+      - bash -c "until curl -s http://ui-regression-php-master > /dev/null && curl -s http://ui-regression-php > /dev/null; do sleep 2; done"
+      - sudo -u pptruser node runTests.js || true
+      - echo "The result can be found at https://s3.bitgrid.net/nextcloud-ui-regression/nextcloud/server/${DRONE_PULL_REQUEST}/index.html"
+    shm_size: '1gb'
+    when:
+      matrix:
+        TESTS: ui-regression
+  publish-s3:
+    image: plugins/s3
+    endpoint: https://s3.bitgrid.net/
+    bucket: nextcloud-ui-regression
+    path_style: true
+    source: tests/ui-regression/out/**/*
+    strip_prefix: tests/ui-regression/out/
+    acl: public-read
+    target: ${DRONE_REPO_OWNER}/${DRONE_REPO_NAME}/${DRONE_PULL_REQUEST}
+    secrets: [ aws_access_key_id, aws_secret_access_key ]
+    when:
+      matrix:
+        TESTS: ui-regression
 matrix:
   include:
     - TESTS: checkers
@@ -848,6 +874,7 @@ matrix:
       ENABLE_REDIS_CLUSTER: true
     - TESTS: sqlite-php7.0-webdav-apache
       ENABLE_REDIS: true
+    - TESTS: ui-regression
 
 services:
   cache:
@@ -855,6 +882,27 @@ services:
     when:
       matrix:
        ENABLE_REDIS: true
+  ui-regression-php:
+    image: nextcloudci/server:server-1
+    commands:
+      - . /etc/apache2/envvars
+      - rm -fr /var/www/html
+      - mkdir /var/www/html && cp -rT . /var/www/html && chown -R www-data:www-data /var/www/html
+      - rm -fr /var/www/html/config/config.php /var/www/html/data/*
+      - apache2 -DFOREGROUND
+    when:
+      matrix:
+        TESTS: ui-regression
+  ui-regression-php-master:
+    image: nextcloudci/server:server-1
+    commands:
+      - . /etc/apache2/envvars
+      - rm -fr /var/www/html/config/config.php /var/www/html/data/*
+      - su www-data -c "cd /var/www/html/ && git checkout $DRONE_REPO_BRANCH && git pull && git submodule update"
+      - apache2 -DFOREGROUND 
+    when:
+      matrix:
+        TESTS: ui-regression
   cache-cluster:
     image: morrisjobke/redis-cluster
     when:
diff --git a/.gitignore b/.gitignore
index a11e3a14597..f8721e67fa8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -139,6 +139,9 @@ Vagrantfile
 /tests/autotest*
 /tests/data/lorem-copy.txt
 /tests/data/testimage-copy.png
+/tests/ui-regression/out/
+/tests/ui-regression/node_modules/
+/tests/ui-regression/package-lock.json
 /config/config-autotest-backup.php
 /config/autoconfig.php
 clover.xml
diff --git a/tests/ui-regression/config.js b/tests/ui-regression/config.js
new file mode 100644
index 00000000000..c6519ba289d
--- /dev/null
+++ b/tests/ui-regression/config.js
@@ -0,0 +1,57 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+module.exports = {
+
+	/**
+	 * Define resolutions to be tested when diffing screenshots
+	 */
+	resolutions: [
+		{title: 'mobile', w: 360, h: 480},
+		{title: 'narrow', w: 800, h: 600},
+		{title: 'normal', w: 1024, h: 768},
+		{title: 'wide', w: 1920, h: 1080},
+		{title: 'qhd', w: 2560, h: 1440},
+		{title: 'uhd', w: 3840, h: 2160},
+	],
+
+	/**
+	 * URL that holds the base branch
+	 */
+	urlBase: 'http://ui-regression-php-master/',
+
+	/**
+	 * URL that holds the branch to be diffed
+	 */
+	urlChange: 'http://ui-regression-php/',
+
+	/**
+	 * Path to output directory for screenshot files
+	 */
+	outputDirectory: 'out',
+
+	/**
+	 * Run in headless mode (useful for debugging)
+	 */
+	headless: true,
+
+};
diff --git a/tests/ui-regression/helper.js b/tests/ui-regression/helper.js
new file mode 100644
index 00000000000..7168c80585b
--- /dev/null
+++ b/tests/ui-regression/helper.js
@@ -0,0 +1,207 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const puppeteer = require('puppeteer');
+const pixelmatch = require('pixelmatch');
+const expect = require('chai').expect;
+const PNG = require('pngjs2').PNG;
+const fs = require('fs');
+const config = require('./config.js');
+
+
+module.exports = {
+	browser: null,
+	pageBase: null,
+	pageCompare: null,
+	init: async function (test) {
+		this._outputDirectory = `${config.outputDirectory}/${test.title}`;
+		if (!fs.existsSync(config.outputDirectory)) fs.mkdirSync(config.outputDirectory);
+		if (!fs.existsSync(this._outputDirectory)) fs.mkdirSync(this._outputDirectory);
+		await this.resetBrowser();
+	},
+	exit: async function () {
+		await this.browser.close();
+	},
+	resetBrowser: async function () {
+		if (this.browser) {
+			await this.browser.close();
+		}
+		this.browser = await puppeteer.launch({
+			args: ['--no-sandbox', '--disable-setuid-sandbox'],
+			headless: config.headless
+		});
+		this.pageBase = await this.browser.newPage();
+		this.pageCompare = await this.browser.newPage();
+		this.pageBase.setDefaultNavigationTimeout(60000);
+		this.pageCompare.setDefaultNavigationTimeout(60000);
+	},
+
+	login: async function (test) {
+		test.timeout(20000);
+		await this.resetBrowser();
+		await Promise.all([
+			this.performLogin(this.pageBase, config.urlBase),
+			this.performLogin(this.pageCompare, config.urlChange)
+		]);
+	},
+
+	performLogin: async function (page, baseUrl) {
+		await page.goto(baseUrl + '/index.php/login', {waitUntil: 'networkidle0'});
+		await page.type('#user', 'admin');
+		await page.type('#password', 'admin');
+		const inputElement = await page.$('input[type=submit]');
+		inputElement.click();
+		return await page.waitForNavigation({waitUntil: 'networkidle0'});
+	},
+
+	takeAndCompare: async function (test, route, action, options) {
+		// use Promise.all
+		if (options === undefined)
+			options = {};
+		if (options.waitUntil === undefined) {
+			options.waitUntil = 'networkidle0';
+		}
+		if (options.viewport) {
+			if (options.viewport.scale === undefined) {
+				options.viewport.scale = 1;
+			}
+			await Promise.all([
+				this.pageBase.setViewport({
+					width: options.viewport.w,
+					height: options.viewport.h,
+					deviceScaleFactor: options.viewport.scale
+				}),
+				this.pageCompare.setViewport({
+					width: options.viewport.w,
+					height: options.viewport.h,
+					deviceScaleFactor: options.viewport.scale
+				})
+			]);
+		}
+		let fileName = test.test.title
+		if (route !== undefined) {
+			await Promise.all([
+				this.pageBase.goto(`${config.urlBase}${route}`, {waitUntil: options.waitUntil}),
+				this.pageCompare.goto(`${config.urlChange}${route}`, {waitUntil: options.waitUntil})
+			]);
+		}
+		var failed = null;
+		try {
+			await Promise.all([
+				action(this.pageBase),
+				action(this.pageCompare)
+			]);
+		} catch (err) {
+			failed = err;
+		}
+		await this.delay(500);
+		await Promise.all([
+			this.pageBase.screenshot({
+				path: `${this._outputDirectory}/${fileName}.base.png`,
+				fullPage: false,
+			}),
+			this.pageCompare.screenshot({
+				path: `${this._outputDirectory}/${fileName}.change.png`,
+				fullPage: false
+			})
+		]);
+
+		if (options.runOnly === true) {
+			fs.unlinkSync(`${this._outputDirectory}/${fileName}.base.png`);
+			fs.renameSync(`${this._outputDirectory}/${fileName}.change.png`, `${this._outputDirectory}/${fileName}.png`);
+		}
+
+		return new Promise(async (resolve, reject) => {
+			try {
+				if (options.runOnly !== true) {
+					await this.compareScreenshots(fileName);
+				}
+			} catch (err) {
+				if (failed) {
+					console.log('Failure during takeAndCompare action callback');
+					console.log(failed);
+				}
+				console.log('Failure when comparing images');
+				return reject(err);
+			}
+			if (options.runOnly !== true && failed) {
+				console.log('Failure during takeAndCompare action callback');
+				console.log(failed);
+				failed.failedAction = true;
+				return reject(failed);
+			}
+			return resolve();
+		});
+	},
+
+	compareScreenshots: function (fileName) {
+		let self = this;
+		return new Promise((resolve, reject) => {
+			const img1 = fs.createReadStream(`${self._outputDirectory}/${fileName}.base.png`).pipe(new PNG()).on('parsed', doneReading);
+			const img2 = fs.createReadStream(`${self._outputDirectory}/${fileName}.change.png`).pipe(new PNG()).on('parsed', doneReading);
+
+			let filesRead = 0;
+
+			function doneReading () {
+				// Wait until both files are read.
+				if (++filesRead < 2) return;
+
+				// The files should be the same size.
+				expect(img1.width, 'image widths are the same').equal(img2.width);
+				expect(img1.height, 'image heights are the same').equal(img2.height);
+
+				// Do the visual diff.
+				const diff = new PNG({width: img1.width, height: img2.height});
+				const numDiffPixels = pixelmatch(
+					img1.data, img2.data, diff.data, img1.width, img1.height,
+					{threshold: 0.3});
+				if (numDiffPixels > 0) {
+					diff.pack().pipe(fs.createWriteStream(`${self._outputDirectory}/${fileName}.diff.png`));
+				} else {
+					fs.unlinkSync(`${self._outputDirectory}/${fileName}.base.png`);
+					fs.renameSync(`${self._outputDirectory}/${fileName}.change.png`, `${self._outputDirectory}/${fileName}.png`);
+				}
+
+				// The files should look the same.
+				expect(numDiffPixels, 'number of different pixels').equal(0);
+				resolve();
+			}
+		});
+	},
+	/**
+	 * Helper function to wait
+	 * to make sure that initial animations are done
+	 */
+	delay: async function (timeout) {
+		return new Promise((resolve) => {
+			setTimeout(resolve, timeout);
+		});
+	},
+
+	childOfClassByText: async function (page, classname, text) {
+		return page.$x('//*[contains(concat(" ", normalize-space(@class), " "), " ' + classname + ' ")]//text()[normalize-space() = \'' + text + '\']/..');
+	},
+
+	childOfIdByText: async function (page, classname, text) {
+		return page.$x('//*[contains(concat(" ", normalize-space(@id), " "), " ' + classname + ' ")]//text()[normalize-space() = \'' + text + '\']/..');
+	}
+};
diff --git a/tests/ui-regression/out/index.html b/tests/ui-regression/out/index.html
new file mode 100644
index 00000000000..a94dae13445
--- /dev/null
+++ b/tests/ui-regression/out/index.html
@@ -0,0 +1,219 @@
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous" />
+	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" />
+	<script src="https://code.jquery.com/jquery-3.3.1.min.js" integrity="sha256-FgpCb/KJQlLNfOu91ta32o/NMZxltwRo8QtmkMRdAu8=" crossorigin="anonymous"></script>
+	<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
+	<title>Nextcloud UI regression tests</title>
+	<style>
+
+		h2 {
+			margin-top: 40px;
+			margin-bottom: 20px;
+		}
+		.error {
+			color: #aa0000;
+		}
+		.success {
+			color: #00aa00;
+		}
+		.success img {
+			display: none;
+			width: 100px;
+		}
+		.success pre {
+			display: none;
+		}
+		.test-result h3 span {
+			width: 40px;
+		}
+		.test-result {
+			padding: 20px;
+		}
+		img {
+			max-width: 33%;
+			padding: 10px;
+			background-color: #eee;
+			margin: 0;
+		}
+		.overview ul {
+			position: fixed;
+			max-width: inherit;
+			margin: 0;
+			padding: 0;
+		}
+		ul li {
+			list-style-type: none;
+			padding: 3px;
+		}
+		ul a:first-child {
+			width: 100%;
+			display: inline-block;
+		}
+		ul span {
+			width: 16px;
+			height: 16px;
+			margin: 1px;
+			display: inline-block;
+		}
+		span.fa-check {
+			color: green;
+		}
+		span.fa-times {
+			color: red;
+		}
+		.navbar a {
+			color: #fff;
+		}
+
+		.fade-enter-active, .fade-leave-active {
+			transition: opacity .5s;
+		}
+		.fade-enter, .fade-leave-to {
+			opacity: 0;
+		}
+	</style>
+</head>
+
+<body>
+<div id="app">
+<nav class="navbar navbar-expand-md navbar-dark bg-dark sticky-top">
+	<div class="container">
+	<a class="navbar-brand" href="#">Nextcloud UI regression test</a>
+	<a class="nav-link" :href="config.repoUrl">{{config.repoUrl}}</a>
+	<a class="nav-link" :href="config.repoUrl + '/pull/' + config.pr">#{{ config.pr }}</span></a>
+	</div>
+</nav>
+
+<main role="main" class="container-fluid">
+	<div class="row">
+		<div class="col-md-2 overview">
+			<ul>
+				<li v-for="suite in config.tests" v-if="result[suite]">
+					<a :href="'#' + suite">{{ suite }}</a>
+					<a v-for="test in result[suite].tests" :href="test.fullTitle | convertToAnchor" :title="test.fullTitle">
+						<span class="fa fa-times" v-if="Object.keys(test.err).length > 0"></span>
+						<span class="fa fa-check" v-else></span>
+					</a>
+				</li>
+			</ul>
+		</div>
+		<div class="col-md-10" id="container">
+			<div v-for="suite in config.tests" v-if="result[suite]">
+				<h2 :id="suite | convertToId">{{ suite }} <span>{{ result[suite].passes.length }}/{{ result[suite].tests.length }}</span></h2>
+				<test-result v-for="test in result[suite].tests" :key="test.fullTitle" :suite="suite" :test="test"></test-result>
+			</div>
+		</div>
+	</div>
+</main>
+</div>
+
+<script type="text/x-template" id="test-result-template">
+	<div class="test-result" :id="test.fullTitle | convertToId">
+		<h3 :class="{ error: Object.keys(test.err).length > 0, success: Object.keys(test.err).length == 0}"
+			v-on:click="hidden === undefined ? hidden = false : hidden = !hidden">
+			<span class="fa fa-times" v-if="Object.keys(test.err).length > 0"></span>
+			<span class="fa fa-check" v-else></span>
+			{{ test.title }}
+			<i v-if="test.duration">{{ test.duration }}ms</i>
+		</h3>
+		<transition name="fade">
+		<div v-if="(hidden === undefined && Object.keys(test.err).length > 0) || hidden === false">
+			<div v-if="Object.keys(test.err).length > 0 && !test.err.failedAction">
+				<a :href="getImagePath('.base')"><img :src="getImagePath('.base')"  /></a>
+				<a :href="getImagePath('.diff')"><img :src="getImagePath('.diff')"  /></a>
+				<a :href="getImagePath('.change')"><img :src="getImagePath('.change')"  /></a>
+			</div>
+			<div v-else>
+				<a :href="getImagePath('')"><img :src="getImagePath('')"  /></a>
+			</div>
+			<pre>{{ jsonData }}</pre>
+		</div>
+		</transition>
+	</div>
+</script>
+
+<script>
+
+	Vue.filter('convertToId', function (id) {
+		return id.replace(/\W/g,'_');
+	});
+
+	Vue.filter('convertToAnchor', function (id) {
+		return '#' + id.replace(/\W/g,'_');
+	});
+
+	Vue.component('test-result', {
+		template: '#test-result-template',
+		props: ['test', 'suite'],
+		data: function () {
+			return {
+				hidden: undefined
+			}
+		},
+		computed: {
+			jsonData: function() {
+				return JSON.stringify(this.test, null, 2)
+			}
+		},
+		methods: {
+			getImagePath: function(type) {
+				return this.suite + '/' + this.test.title + type + '.png';
+			}
+		}
+	});
+
+	var app = new Vue({
+		el: '#app',
+		data: {
+			message: 'Hello Vue!',
+			config: {},
+			result: {
+				login: {}
+			},
+		},
+		created: function() {
+			this.fetchConfig();
+		},
+		methods: {
+			fetchConfig: function() {
+				var request = new XMLHttpRequest();
+				request.open('GET', 'config.json', true);
+
+				request.onload = function() {
+					if (request.status >= 200 && request.status < 400) {
+						app.config = JSON.parse(request.responseText);
+						app.config.tests.forEach(function(item, i){
+							app.fetchResults(item);
+						});
+					}
+				};
+
+				request.onerror = function() {
+				};
+
+				request.send();
+			},
+			fetchResults: function(suite) {
+				var request = new XMLHttpRequest();
+				request.open('GET', suite + '.json', true);
+
+				request.onload = function() {
+					if (request.status >= 200 && request.status < 400) {
+						Vue.set(app.result, suite, JSON.parse(request.responseText));
+					}
+				};
+
+				request.onerror = function() {
+				};
+
+				request.send();
+			}
+		}
+	});
+
+</script>
+</body>
+</html>
diff --git a/tests/ui-regression/package.json b/tests/ui-regression/package.json
new file mode 100644
index 00000000000..979dfed3dec
--- /dev/null
+++ b/tests/ui-regression/package.json
@@ -0,0 +1,21 @@
+{
+    "name": "ui-regression",
+    "version": "1.0.0",
+    "description": "",
+    "main": "index.js",
+    "scripts": {
+        "test": "mocha test/"
+    },
+    "author": "",
+    "dependencies": {
+        "fs": "0.0.1-security",
+        "chai": "^4.1.2",
+        "mocha": "^5.0.0",
+        "mocha-json-report": "0.0.2",
+        "pixelmatch": "^4.0.2",
+        "png-js": "^0.1.1",
+        "pngjs2": "^2.0.0",
+        "polyserve": "^0.23.0",
+        "puppeteer": "^1.0.0"
+    }
+}
diff --git a/tests/ui-regression/runTests.js b/tests/ui-regression/runTests.js
new file mode 100644
index 00000000000..4eb94f79347
--- /dev/null
+++ b/tests/ui-regression/runTests.js
@@ -0,0 +1,129 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const fs = require('fs')
+const Mocha = require('mocha')
+
+const testFolder = './test/'
+
+
+var tests = [
+	'install',
+	'login',
+	'files',
+	'public',
+	'settings',
+	'apps',
+]
+
+var args = process.argv.slice(2);
+if (args.length > 0) {
+	tests = args
+}
+
+var config = {
+	tests: tests,
+	pr: process.env.DRONE_PULL_REQUEST,
+	repoUrl: process.env.DRONE_REPO_LINK,
+};
+
+console.log('=> Write test config');
+console.log(config);
+fs.writeFile('out/config.json', JSON.stringify(config), 'utf8', () => {});
+
+var mocha = new Mocha({
+	timeout: 60000
+});
+let result = {};
+
+tests.forEach(async function (test) {
+	mocha.addFile('./test/' + test + 'Spec.js')
+	result[test] = {
+		failures: [],
+		passes: [],
+		tests: [],
+		pending: [],
+		stats: {}
+	}
+
+});
+
+// fixme fail if installation failed
+// write json to file
+
+function clean (test) {
+	return {
+		title: test.title,
+		fullTitle: test.fullTitle(),
+		duration: test.duration,
+		currentRetry: test.currentRetry(),
+		failedAction: test.failedAction,
+		err: errorJSON(test.err || {})
+	};
+}
+
+function errorJSON (err) {
+	var res = {};
+	Object.getOwnPropertyNames(err).forEach(function (key) {
+		res[key] = err[key];
+	}, err);
+	return res;
+}
+
+mocha.run()
+	.on('test', function (test) {
+	})
+	.on('suite end', function(suite) {
+		if (result[suite.title] === undefined)
+			return;
+		result[suite.title].stats = suite.stats;
+	})
+	.on('test end', function (test) {
+		result[test.parent.title].tests.push(test);
+	})
+	.on('pass', function (test) {
+		result[test.parent.title].passes.push(test);
+	})
+	.on('fail', function (test) {
+		result[test.parent.title].failures.push(test);
+	})
+	.on('pending', function (test) {
+		result[test.parent.title].pending.push(test);
+	})
+	.on('end', function () {
+		tests.forEach(function (test) {
+			var json = JSON.stringify({
+				stats: result[test].stats,
+				tests: result[test].tests.map(clean),
+				pending: result[test].pending.map(clean),
+				failures: result[test].failures.map(clean),
+				passes: result[test].passes.map(clean)
+			}, null, 2);
+			fs.writeFile(`out/${test}.json`, json, 'utf8', function () {
+				console.log(`Written test result to out/${test}.json`)
+			});
+		});
+
+		var errorMessage = 'This PR introduces some UI differences, please check at {LINK}, if there are regressions based on the changes.'
+		fs.writeFile('out/GITHUB_COMMENT', errorMessage, 'utf8', () => {});
+	});
+
diff --git a/tests/ui-regression/test/appsSpec.js b/tests/ui-regression/test/appsSpec.js
new file mode 100644
index 00000000000..b5e5a889e41
--- /dev/null
+++ b/tests/ui-regression/test/appsSpec.js
@@ -0,0 +1,60 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('apps', function () {
+
+	before(async () => {
+    await helper.init(this) 
+    await helper.login(this) 
+  });
+	after(async () => await helper.exit());
+
+	config.resolutions.forEach(function (resolution) {
+		it('apps.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, 'index.php/settings/apps', async function (page) {
+				await page.waitForSelector('#apps-list .section', {timeout: 5000});
+				await page.waitFor(500);
+			}, {viewport: resolution, waitUntil: 'networkidle2'});
+		});
+
+		['installed', 'enabled', 'disabled', 'app-bundles'].forEach(function(endpoint) {
+			it('apps.' + endpoint + '.' + resolution.title, async function () {
+				return helper.takeAndCompare(this, undefined, async function (page) {
+					try {
+						await page.waitForSelector('#app-navigation-toggle', {
+							visible: true,
+							timeout: 1000,
+						}).then((element) => element.click())
+					} catch (err) {}
+					await helper.delay(500);
+					await page.click('li#app-category-' + endpoint + ' a');
+					await helper.delay(500);
+					await page.waitForSelector('#app-content:not(.icon-loading)');
+				}, {viewport: resolution});
+			});
+		});
+	});
+
+});
diff --git a/tests/ui-regression/test/filesSpec.js b/tests/ui-regression/test/filesSpec.js
new file mode 100644
index 00000000000..be507390f4e
--- /dev/null
+++ b/tests/ui-regression/test/filesSpec.js
@@ -0,0 +1,101 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const puppeteer = require('puppeteer');
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('files', function () {
+
+	before(async () => {
+		await helper.init(this)
+		await helper.login(this)
+	});
+	after(async () => await helper.exit());
+
+	config.resolutions.forEach(function (resolution) {
+
+		it('file-sidebar-share.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) {
+				let element = await page.$('[data-file="welcome.txt"] .action-share');
+				await element.click('[data-file="welcome.txt"] .action-share');
+				await page.waitForSelector('.shareWithField');
+				await helper.delay(500);
+				await page.$eval('body', e => { $('.shareWithField').blur() });
+			}, {viewport: resolution, waitUntil: 'networkidle2'});
+		});
+		it('file-popover.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) {
+				await page.click('[data-file=\'welcome.txt\'] .action-menu');
+				await page.waitForSelector('.fileActionsMenu');
+			}, {viewport: resolution, waitUntil: 'networkidle2'});
+		});
+		it('file-sidebar-details.' + resolution.title, async function() {
+			return helper.takeAndCompare(this, undefined, async function (page) {
+				await page.click('[data-file=\'welcome.txt\'] .fileActionsMenu [data-action=\'Details\']');
+				await page.waitForSelector('#commentsTabView');
+				await helper.delay(500); // wait for animation
+			});
+		});
+		it('file-sidebar-details-sharing.' + resolution.title, async function() {
+			return helper.takeAndCompare(this, undefined, async function (page) {
+				let tab = await helper.childOfClassByText(page, 'tabHeaders', 'Sharing');
+				tab[0].click();
+				await page.waitForSelector('input.shareWithField');
+				await helper.delay(500); // wait for animation
+				await page.$eval('body', e => { $('.shareWithField').blur() });
+			});
+		});
+		it('file-sidebar-details-versions.' + resolution.title, async function() {
+			return helper.takeAndCompare(this, undefined, async function (page) {
+				let tab = await helper.childOfClassByText(page, 'tabHeaders', 'Versions');
+				tab[0].click();
+				await helper.delay(100); // wait for animation
+			});
+		});
+		it('file-popover.favorite.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) {
+				await page.click('[data-file=\'welcome.txt\'] .action-menu');
+				await page.waitForSelector('.fileActionsMenu')
+				await page.click('[data-file=\'welcome.txt\'] .fileActionsMenu [data-action=\'Favorite\']');;
+			}, {viewport: resolution, waitUntil: 'networkidle2'});
+		});
+
+		it('file-favorites.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, 'index.php/apps/files', async function (page) {
+				try {
+					await page.waitForSelector('#app-navigation-toggle', {
+						visible: true,
+						timeout: 1000,
+					}).then((element) => element.click())
+				} catch (err) {}
+				await page.click('#app-navigation [data-id=\'favorites\'] a');
+				await helper.delay(500); // wait for animation
+			}, {viewport: resolution, waitUntil: 'networkidle2'});
+		});
+
+
+	});
+
+
+
+});
diff --git a/tests/ui-regression/test/installSpec.js b/tests/ui-regression/test/installSpec.js
new file mode 100644
index 00000000000..02577883aae
--- /dev/null
+++ b/tests/ui-regression/test/installSpec.js
@@ -0,0 +1,75 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('install', function () {
+
+	before(async () => await helper.init(this));
+	after(async () => await helper.exit());
+
+	config.resolutions.forEach(function (resolution) {
+		it('show-page.' + resolution.title, async function () {
+			// (test, route, prepare, action, options
+			return helper.takeAndCompare(this, '/index.php', async (page) => {
+				await helper.delay(100);
+				await page.$eval('body', function (e) {
+					$('#adminlogin').blur();
+				});
+				await helper.delay(100);
+			}, { waitUntil: 'networkidle0', viewport: resolution});
+		});
+
+		it('show-advanced.' + resolution.title, async function () {
+			// (test, route, prepare, action, options
+			return helper.takeAndCompare(this, undefined, async (page) => {
+				await page.click('#showAdvanced');
+				await helper.delay(500);
+			});
+		});
+		it('show-advanced-mysql.' + resolution.title, async function () {
+			// (test, route, prepare, action, options
+			return helper.takeAndCompare(this, undefined, async (page) => {
+				await page.click('label.mysql');
+				await helper.delay(500);
+			});
+		});
+	});
+
+	it('runs', async function () {
+		this.timeout(5*60*1000);
+		helper.pageBase.setDefaultNavigationTimeout(5*60*1000);
+		helper.pageCompare.setDefaultNavigationTimeout(5*60*1000);
+		// just run for one resolution since we can only install once
+		return helper.takeAndCompare(this, '/index.php',  async function (page) {
+			const login = await page.type('#adminlogin', 'admin');
+			const password = await page.type('#adminpass', 'admin');
+			const inputElement = await page.$('input[type=submit]');
+			await inputElement.click();
+			await page.waitForNavigation({waitUntil: 'networkidle0'});
+			helper.pageBase.setDefaultNavigationTimeout(60000);
+			helper.pageCompare.setDefaultNavigationTimeout(60000);
+		}, { waitUntil: 'networkidle0', viewport: {w: 1920, h: 1080}});
+	});
+
+});
diff --git a/tests/ui-regression/test/loginSpec.js b/tests/ui-regression/test/loginSpec.js
new file mode 100644
index 00000000000..23f86737a76
--- /dev/null
+++ b/tests/ui-regression/test/loginSpec.js
@@ -0,0 +1,75 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('login', function () {
+
+	before(async () => await helper.init(this));
+	after(async () => await helper.exit());
+
+	/**
+	 * Test login page rendering
+	 */
+	config.resolutions.forEach(function (resolution) {
+		it('login-page.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/', async (page) => {
+				// make sure the cursor is not blinking in the login field
+				await page.$eval('body', function (e) {
+					$('#user').blur();
+				});
+				return await helper.delay(100);
+			}, {viewport: resolution});
+		});
+
+		it('login-page.forgot.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, undefined, async (page) => {
+				const lostPassword = await page.$('#lost-password');
+				await lostPassword.click();
+				await helper.delay(500);
+				await page.$eval('body', function (e) {
+					$('#user').blur();
+				});
+			}, {viewport: resolution});
+		});
+	});
+
+	/**
+	 * Perform login
+	 */
+	config.resolutions.forEach(function (resolution) {
+		it('login-success.' + resolution.title, async function () {
+			this.timeout(30000);
+			await helper.resetBrowser();
+			return helper.takeAndCompare(this, '/', async function (page) {
+				await page.type('#user', 'admin');
+				await page.type('#password', 'admin');
+				const inputElement = await page.$('input[type=submit]');
+				await inputElement.click();
+				await page.waitForNavigation({waitUntil: 'networkidle0'});
+				return await helper.delay(100);
+			}, {viewport: resolution});
+		})
+	});
+
+});
diff --git a/tests/ui-regression/test/publicSpec.js b/tests/ui-regression/test/publicSpec.js
new file mode 100644
index 00000000000..843f8f50cef
--- /dev/null
+++ b/tests/ui-regression/test/publicSpec.js
@@ -0,0 +1,102 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const puppeteer = require('puppeteer');
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('public', function () {
+
+	before(async () => {
+		await helper.init(this)
+		await helper.login(this)
+	});
+	after(async () => await helper.exit());
+
+	/**
+	 * Test invalid file share rendering
+	 */
+	config.resolutions.forEach(function (resolution) {
+		it('file-share-invalid.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/index.php/s/invalid', async function () {
+			}, {waitUntil: 'networkidle2', viewport: resolution});
+		});
+	});
+
+	/**
+	 * Share a file via public link
+	 */
+
+	var shareLink = {};
+	it('file-share-link', async function () {
+		return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) {
+			const element = await page.$('[data-file="welcome.txt"] .action-share');
+			await element.click('[data-file="welcome.txt"] .action-share');
+			await page.waitForSelector('input.linkCheckbox');
+			const linkCheckbox = await page.$('.linkShareView label');
+			await Promise.all([
+				linkCheckbox.click(),
+				page.waitForSelector('.linkText')
+			]);
+			await helper.delay(500);
+			const text = await page.waitForSelector('.linkText');
+			const link = await (await text.getProperty('value')).jsonValue();
+			shareLink[page.url()] = link;
+			return await helper.delay(500);
+		}, {
+			runOnly: true,
+			waitUntil: 'networkidle2',
+			viewport: {w: 1920, h: 1080}
+		});
+	});
+
+	config.resolutions.forEach(function (resolution) {
+		it('file-share-valid.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) {
+				await page.goto(shareLink[page.url()]);
+				await helper.delay(500);
+			}, {waitUntil: 'networkidle2', viewport: resolution});
+		});
+		it('file-share-valid-actions.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, undefined, async function (page) {
+				const moreButton = await page.waitForSelector('#header-secondary-action');
+				await moreButton.click();
+				await page.evaluate((data) => {
+					return document.querySelector('#directLink').value = 'http://nextcloud.example.com/';
+				});
+				await helper.delay(500);
+			}, {waitUntil: 'networkidle2', viewport: resolution});
+		});
+	});
+
+	it('file-unshare', async function () {
+		return helper.takeAndCompare(this, '/index.php/apps/files', async function (page) {
+			const element = await page.$('[data-file="welcome.txt"] .action-share');
+			await element.click('[data-file="welcome.txt"] .action-share');
+			await page.waitForSelector('input.linkCheckbox');
+			const linkCheckbox = await page.$('.linkShareView label');
+			await linkCheckbox.click();
+			await helper.delay(500);
+		}, { waitUntil: 'networkidle2', viewport: {w: 1920, h:1080}});
+	});
+
+});
diff --git a/tests/ui-regression/test/settingsSpec.js b/tests/ui-regression/test/settingsSpec.js
new file mode 100644
index 00000000000..560218c80f8
--- /dev/null
+++ b/tests/ui-regression/test/settingsSpec.js
@@ -0,0 +1,76 @@
+/**
+ * @copyright 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author 2018 Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+const helper = require('../helper.js');
+const config = require('../config.js');
+
+describe('settings', function () {
+
+	before(async () => {
+    await helper.init(this) 
+    await helper.login(this) 
+  });
+	after(async () => await helper.exit());
+
+	config.resolutions.forEach(function (resolution) {
+		it('personal.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/index.php/settings/user', async function (page) {
+			}, {viewport: resolution});
+		});
+
+		it('admin.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/index.php/settings/admin', async function (page) {
+			}, {viewport: resolution});
+		});
+
+		['sharing', 'security', 'theming', 'encryption', 'additional', 'tips-tricks'].forEach(function(endpoint) {
+			it('admin.' + endpoint + '.' + resolution.title, async function () {
+				return helper.takeAndCompare(this, '/index.php/settings/admin/' + endpoint, async function (page) {
+				}, {viewport: resolution, waitUntil: 'networkidle2'});
+			});
+		});
+
+		it('usermanagement.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, '/index.php/settings/users', async function (page) {
+			}, {viewport: resolution});
+		});
+
+		it('usermanagement.add.' + resolution.title, async function () {
+			return helper.takeAndCompare(this, undefined, async function (page) {
+				try {
+					await page.waitForSelector('#app-navigation-toggle', {
+						visible: true,
+						timeout: 1000,
+					}).then((element) => element.click())
+				} catch (err) {}
+				let newUserButton = await page.waitForSelector('#new-user-button');
+				await newUserButton.click();
+				await helper.delay(200);
+				await page.$eval('body', function (e) {
+					$('#newusername').blur();
+				})
+				await helper.delay(100);
+			}, {viewport: resolution});
+		});
+
+	});
+});