Add basic performance testing via Cypress (#8586)

.github/workflows/element-build-and-test.yaml

@@ -88,6 +88,20 @@ jobs:
             cypress/videos
             cypress/synapselogs
 
+      - name: Store benchmark result
+        if: github.ref == 'refs/heads/develop'
+        uses: matrix-org/github-action-benchmark@jsperfentry-1
+        with:
+          name: Cypress measurements
+          tool: 'jsperformanceentry'
+          output-file-path: cypress/performance/measurements.json
+          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
+          benchmark-data-dir-path: cypress/bench
+          fail-on-alert: false
+          comment-on-alert: false
+          github-token: ${{ secrets.DEPLOY_GH_PAGES }}
+          auto-push: ${{ github.ref == 'refs/heads/develop' }}
+
   app-tests:
     name: Element Web Integration Tests
     runs-on: ubuntu-latest

.github/workflows/end-to-end-tests.yaml

@@ -40,20 +40,18 @@ jobs:
             test/end-to-end-tests/synapse/installations/consent/homeserver.log
             retention-days: 14
 
-      - name: Download previous benchmark data
-        uses: actions/cache@v1
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
-
       - name: Store benchmark result
+        if: github.ref == 'refs/heads/develop'
         uses: matrix-org/github-action-benchmark@jsperfentry-1
         with:
           tool: 'jsperformanceentry'
           output-file-path: test/end-to-end-tests/performance-entries.json
+          # This is the default dashboard path. It's included here anyway to
+          # make the difference from the Cypress variant in
+          # `element-build-and-test.yaml` more obvious.
+          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/dev/bench/
+          benchmark-data-dir-path: dev/bench
           fail-on-alert: false
           comment-on-alert: false
-          # Only temporary to monitor where failures occur
-          alert-comment-cc-users: '@gsouquet'
           github-token: ${{ secrets.DEPLOY_GH_PAGES }}
           auto-push: ${{ github.ref == 'refs/heads/develop' }}

.gitignore

@@ -27,3 +27,4 @@ package-lock.json
 # These could have files in them but don't currently
 # Cypress will still auto-create them though...
 /cypress/fixtures
+/cypress/performance

cypress/global.d.ts

@@ -18,6 +18,7 @@ import "matrix-js-sdk/src/@types/global";
 import type { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
 import type { RoomMemberEvent } from "matrix-js-sdk/src/models/room-member";
 import type { MatrixDispatcher } from "../src/dispatcher/dispatcher";
+import type PerformanceMonitor from "../src/performance";
 
 declare global {
     // eslint-disable-next-line @typescript-eslint/no-namespace
@@ -27,6 +28,7 @@ declare global {
                 matrixClient?: MatrixClient;
             };
             mxDispatcher: MatrixDispatcher;
+            mxPerformanceMonitor: PerformanceMonitor;
             beforeReload?: boolean; // for detecting reloads
             // Partial type for the matrix-js-sdk module, exported by browser-matrix
             matrixcs: {
@@ -38,7 +40,11 @@ declare global {
     }
 
     interface Window {
-        mxDispatcher: MatrixDispatcher; // to appease the MatrixDispatcher import
+        // to appease the MatrixDispatcher import
+        mxDispatcher: MatrixDispatcher;
+        // to appease the PerformanceMonitor import
+        mxPerformanceMonitor: PerformanceMonitor;
+        mxPerformanceEntryNames: any;
     }
 }
 

cypress/integration/1-register/register.spec.ts

@@ -42,11 +42,15 @@ describe("Registration", () => {
         cy.get("#mx_RegistrationForm_username").type("alice");
         cy.get("#mx_RegistrationForm_password").type("totally a great password");
         cy.get("#mx_RegistrationForm_passwordConfirm").type("totally a great password");
+        cy.startMeasuring("create-account");
         cy.get(".mx_Login_submit").click();
 
         cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
+        cy.stopMeasuring("create-account");
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
+        cy.startMeasuring("from-submit-to-home");
         cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
         cy.url().should('contain', '/#/home');
+        cy.stopMeasuring("from-submit-to-home");
     });
 });

cypress/integration/2-login/login.spec.ts

@@ -49,9 +49,11 @@ describe("Login", () => {
 
             cy.get("#mx_LoginForm_username").type(username);
             cy.get("#mx_LoginForm_password").type(password);
+            cy.startMeasuring("from-submit-to-home");
             cy.get(".mx_Login_submit").click();
 
             cy.url().should('contain', '/#/home');
+            cy.stopMeasuring("from-submit-to-home");
         });
     });
 });

cypress/integration/4-create-room/create-room.spec.ts

@@ -54,10 +54,12 @@ describe("Create Room", () => {
             // Fill room address
             cy.get('[label="Room address"]').type("test-room-1");
             // Submit
+            cy.startMeasuring("from-submit-to-room");
             cy.get(".mx_Dialog_primary").click();
         });
 
         cy.url().should("contain", "/#/room/#test-room-1:localhost");
+        cy.stopMeasuring("from-submit-to-room");
         cy.get(".mx_RoomHeader_nametext").contains(name);
         cy.get(".mx_RoomHeader_topic").contains(topic);
     });

cypress/plugins/index.ts

@@ -16,13 +16,15 @@ limitations under the License.
 
 /// <reference types="cypress" />
 
-import { synapseDocker } from "./synapsedocker";
 import PluginEvents = Cypress.PluginEvents;
 import PluginConfigOptions = Cypress.PluginConfigOptions;
+import { performance } from "./performance";
+import { synapseDocker } from "./synapsedocker";
 
 /**
  * @type {Cypress.PluginConfig}
  */
 export default function(on: PluginEvents, config: PluginConfigOptions) {
+    performance(on, config);
     synapseDocker(on, config);
 }

cypress/plugins/performance.ts

@@ -0,0 +1,47 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import * as path from "path";
+import * as fse from "fs-extra";
+
+import PluginEvents = Cypress.PluginEvents;
+import PluginConfigOptions = Cypress.PluginConfigOptions;
+
+// This holds all the performance measurements throughout the run
+let bufferedMeasurements: PerformanceEntry[] = [];
+
+function addMeasurements(measurements: PerformanceEntry[]): void {
+    bufferedMeasurements = bufferedMeasurements.concat(measurements);
+    return null;
+}
+
+async function writeMeasurementsFile() {
+    try {
+        const measurementsPath = path.join("cypress", "performance", "measurements.json");
+        await fse.outputJSON(measurementsPath, bufferedMeasurements, {
+            spaces: 4,
+        });
+    } finally {
+        bufferedMeasurements = [];
+    }
+}
+
+export function performance(on: PluginEvents, config: PluginConfigOptions) {
+    on("task", { addMeasurements });
+    on("after:run", writeMeasurementsFile);
+}

cypress/plugins/synapsedocker/index.ts

@@ -201,7 +201,7 @@ async function synapseStop(id: string): Promise<void> {
     synapses.delete(id);
 
     console.log(`Stopped synapse id ${id}.`);
-    // cypres deliberately fails if you return 'undefined', so
+    // cypress deliberately fails if you return 'undefined', so
     // return null to signal all is well and we've handled the task.
     return null;
 }

cypress/support/bot.ts

@@ -40,7 +40,7 @@ Cypress.Commands.add("getBot", (synapse: SynapseInstance, displayName?: string):
     const username = Cypress._.uniqueId("userId_");
     const password = Cypress._.uniqueId("password_");
     return cy.registerUser(synapse, username, password, displayName).then(credentials => {
-        return cy.window().then(win => {
+        return cy.window({ log: false }).then(win => {
             const cli = new win.matrixcs.MatrixClient({
                 baseUrl: synapse.baseUrl,
                 userId: credentials.userId,

cypress/support/client.ts

@@ -46,11 +46,11 @@ declare global {
 }
 
 Cypress.Commands.add("getClient", (): Chainable<MatrixClient | undefined> => {
-    return cy.window().then(win => win.mxMatrixClientPeg.matrixClient);
+    return cy.window({ log: false }).then(win => win.mxMatrixClientPeg.matrixClient);
 });
 
 Cypress.Commands.add("createRoom", (options: ICreateRoomOpts): Chainable<string> => {
-    return cy.window().then(async win => {
+    return cy.window({ log: false }).then(async win => {
         const cli = win.mxMatrixClientPeg.matrixClient;
         const resp = await cli.createRoom(options);
         const roomId = resp.room_id;

cypress/support/index.ts

@@ -16,6 +16,7 @@ limitations under the License.
 
 /// <reference types="cypress" />
 
+import "./performance";
 import "./synapse";
 import "./login";
 import "./client";

cypress/support/login.ts

@@ -43,7 +43,7 @@ declare global {
 
 Cypress.Commands.add("initTestUser", (synapse: SynapseInstance, displayName: string): Chainable<UserCredentials> => {
     // XXX: work around Cypress not clearing IDB between tests
-    cy.window().then(win => {
+    cy.window({ log: false }).then(win => {
         win.indexedDB.databases().then(databases => {
             databases.forEach(database => {
                 win.indexedDB.deleteDatabase(database.name);
@@ -73,7 +73,7 @@ Cypress.Commands.add("initTestUser", (synapse: SynapseInstance, displayName: str
             },
         });
     }).then(response => {
-        cy.window().then(win => {
+        cy.window({ log: false }).then(win => {
             // Seed the localStorage with the required credentials
             win.localStorage.setItem("mx_hs_url", synapse.baseUrl);
             win.localStorage.setItem("mx_user_id", response.body.user_id);

cypress/support/performance.ts

@@ -0,0 +1,74 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import Chainable = Cypress.Chainable;
+import AUTWindow = Cypress.AUTWindow;
+
+declare global {
+    // eslint-disable-next-line @typescript-eslint/no-namespace
+    namespace Cypress {
+        interface Chainable {
+            /**
+             * Start measuring the duration of some task.
+             * @param task The task name.
+             */
+            startMeasuring(task: string): Chainable<AUTWindow>;
+            /**
+             * Stop measuring the duration of some task.
+             * The duration is reported in the Cypress log.
+             * @param task The task name.
+             */
+            stopMeasuring(task: string): Chainable<AUTWindow>;
+        }
+    }
+}
+
+function getPrefix(task: string): string {
+    return `cy:${Cypress.spec.name.split(".")[0]}:${task}`;
+}
+
+function startMeasuring(task: string): Chainable<AUTWindow> {
+    return cy.window({ log: false }).then((win) => {
+        win.mxPerformanceMonitor.start(getPrefix(task));
+    });
+}
+
+function stopMeasuring(task: string): Chainable<AUTWindow> {
+    return cy.window({ log: false }).then((win) => {
+        const measure = win.mxPerformanceMonitor.stop(getPrefix(task));
+        cy.log(`**${task}** ${measure.duration} ms`);
+    });
+}
+
+Cypress.Commands.add("startMeasuring", startMeasuring);
+Cypress.Commands.add("stopMeasuring", stopMeasuring);
+
+Cypress.on("window:before:unload", (event: BeforeUnloadEvent) => {
+    const doc = event.target as Document;
+    if (doc.location.href === "about:blank") return;
+    const win = doc.defaultView as AUTWindow;
+    if (!win.mxPerformanceMonitor) return;
+    const entries = win.mxPerformanceMonitor.getEntries().filter(entry => {
+        return entry.name.startsWith("cy:");
+    });
+    if (!entries || entries.length === 0) return;
+    cy.task("addMeasurements", entries);
+});
+
+// Needed to make this file a module
+export { };

cypress/support/synapse.ts

@@ -63,7 +63,7 @@ function startSynapse(template: string): Chainable<SynapseInstance> {
 function stopSynapse(synapse?: SynapseInstance): Chainable<AUTWindow> {
     if (!synapse) return;
     // Navigate away from app to stop the background network requests which will race with Synapse shutting down
-    return cy.window().then((win) => {
+    return cy.window({ log: false }).then((win) => {
         win.location.href = 'about:blank';
         cy.task("synapseStop", synapse.synapseId);
     });

src/performance/index.ts

@@ -71,7 +71,7 @@ export default class PerformanceMonitor {
      * with the start marker
      * @param name Name of the recording
      * @param id Specify an identifier appended to the measurement name
-     * @returns {void}
+     * @returns The measurement
      */
     stop(name: string, id?: string): PerformanceEntry {
         if (!this.supportsPerformanceApi()) {