From e01ac44cfae7d724daa89a916882fd08cc023de0 Mon Sep 17 00:00:00 2001 From: Artur Wierzbicki Date: Wed, 12 Jan 2022 22:15:29 +0400 Subject: [PATCH] Live: performance tests e2e part (#43915) * #41993: live perf tests e2e part * #41993: added bash upgrade instructions * #41993: remove custom feature toggle * #41993: fix typo in 'integrationFolder' --- .gitignore | 2 + .../dashboards/live/4-20hz-panels.json | 399 ++++++++++++++++++ e2e/benchmarks/live/4-20hz-panels.spec.ts | 41 ++ e2e/benchmarks/tsconfig.json | 5 + e2e/custom.ini | 2 + e2e/run-suite | 76 +++- e2e/start-server | 1 + package.json | 1 + .../plugins/benchmark/CDPDataCollector.ts | 136 ++++++ .../plugins/benchmark/DataCollector.ts | 14 + .../cypress/plugins/benchmark/formatting.ts | 138 ++++++ .../cypress/plugins/benchmark/index.ts | 87 ++++ .../cypress/plugins/benchmark/tracelib.d.ts | 15 + packages/grafana-e2e/cypress/plugins/index.js | 5 + .../grafana-e2e/cypress/support/commands.ts | 8 + .../grafana-e2e/cypress/support/index.d.ts | 2 + packages/grafana-e2e/cypress/tsconfig.json | 2 +- packages/grafana-e2e/package.json | 7 + .../grafana-e2e/src/flows/importDashboard.ts | 37 +- .../grafana-e2e/src/flows/importDashboards.ts | 5 +- packages/grafana-e2e/src/index.ts | 2 + packages/grafana-e2e/src/support/benchmark.ts | 81 ++++ yarn.lock | 63 +++ 23 files changed, 1099 insertions(+), 30 deletions(-) create mode 100644 e2e/benchmarks/dashboards/live/4-20hz-panels.json create mode 100644 e2e/benchmarks/live/4-20hz-panels.spec.ts create mode 100644 e2e/benchmarks/tsconfig.json create mode 100644 e2e/custom.ini create mode 100644 packages/grafana-e2e/cypress/plugins/benchmark/CDPDataCollector.ts create mode 100644 packages/grafana-e2e/cypress/plugins/benchmark/DataCollector.ts create mode 100644 packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts create mode 100644 packages/grafana-e2e/cypress/plugins/benchmark/index.ts create mode 100644 packages/grafana-e2e/cypress/plugins/benchmark/tracelib.d.ts create mode 100644 packages/grafana-e2e/src/support/benchmark.ts diff --git a/.gitignore b/.gitignore index b1e2b273bc1..3201f07be2a 100644 --- a/.gitignore +++ b/.gitignore @@ -136,6 +136,8 @@ compilation-stats.json /e2e/**/screenshots !/e2e/**/screenshots/expected/* /e2e/**/videos/* +/e2e/benchmarks/**/results/* +/e2e/benchmarks/**/results # a11y tests /pa11y-ci-results.json diff --git a/e2e/benchmarks/dashboards/live/4-20hz-panels.json b/e2e/benchmarks/dashboards/live/4-20hz-panels.json new file mode 100644 index 00000000000..aed51ce308b --- /dev/null +++ b/e2e/benchmarks/dashboards/live/4-20hz-panels.json @@ -0,0 +1,399 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 82, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 10, + "maxDataPoints": 1500, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "channel": "plugin/testdata/random-20Hz-stream-5", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "measurements", + "refId": "A" + } + ], + "title": "5", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "maxDataPoints": 1500, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "channel": "plugin/testdata/random-20Hz-stream-2", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "measurements", + "refId": "A" + } + ], + "title": "2", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 8, + "maxDataPoints": 1500, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "channel": "plugin/testdata/random-20Hz-stream-4", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "measurements", + "refId": "A" + } + ], + "title": "4", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "maxDataPoints": 1500, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "none" + } + }, + "targets": [ + { + "buffer": 60000, + "channel": "plugin/testdata/random-20Hz-stream", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "filter": { + "fields": [] + }, + "queryType": "measurements", + "refId": "A" + } + ], + "title": "1", + "transparent": true, + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 33, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-60s", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Grafana Live performance benchmarking", + "uid": "S4M5r9cnk", + "version": 9, + "weekStart": "" +} diff --git a/e2e/benchmarks/live/4-20hz-panels.spec.ts b/e2e/benchmarks/live/4-20hz-panels.spec.ts new file mode 100644 index 00000000000..1531d4cc567 --- /dev/null +++ b/e2e/benchmarks/live/4-20hz-panels.spec.ts @@ -0,0 +1,41 @@ +import { e2e } from '@grafana/e2e'; + +type WithGrafanaRuntime = T & { + grafanaRuntime: { + livePerformance: { + start: () => void; + getStats: () => Record; + }; + }; +}; + +const hasGrafanaRuntime = (obj: T): obj is WithGrafanaRuntime => { + return typeof (obj as any)?.grafanaRuntime === 'object'; +}; + +e2e.benchmark({ + name: 'Live performance benchmarking - 4x20hz panels', + dashboard: { + folder: '/dashboards/live', + delayAfterOpening: 1000, + skipPanelValidation: true, + }, + repeat: 10, + duration: 120000, + appStats: { + startCollecting: (window) => { + if (!hasGrafanaRuntime(window)) { + return; + } + + return window.grafanaRuntime.livePerformance.start(); + }, + collect: (window) => { + if (!hasGrafanaRuntime(window)) { + return {}; + } + + return window.grafanaRuntime.livePerformance.getStats() ?? {}; + }, + }, +}); diff --git a/e2e/benchmarks/tsconfig.json b/e2e/benchmarks/tsconfig.json new file mode 100644 index 00000000000..546c7ef03e2 --- /dev/null +++ b/e2e/benchmarks/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts", "../../packages/grafana-e2e/cypress/support/index.d.ts"], + "resolveJsonModule": true +} diff --git a/e2e/custom.ini b/e2e/custom.ini new file mode 100644 index 00000000000..cd4f4c9103d --- /dev/null +++ b/e2e/custom.ini @@ -0,0 +1,2 @@ +[feature_toggles] +enable = diff --git a/e2e/run-suite b/e2e/run-suite index 2c6715562ba..ce9707169c3 100755 --- a/e2e/run-suite +++ b/e2e/run-suite @@ -1,8 +1,14 @@ -#!/bin/bash +#!/usr/bin/env bash set -xeo pipefail . e2e/variables +if ((BASH_VERSINFO[0] < 4)); then + echo "Bash ver >= 4 is needed to run this script" + echo "Please upgrade your bash - run 'brew install bash' if you use Homebrew on MacOS" + exit 1; +fi + HOST=${HOST:-$DEFAULT_HOST} PORT=${PORT:-$DEFAULT_PORT} @@ -12,31 +18,79 @@ args=("$@") CMD="start" PARAMS="" -SLOWMO=0 -URL=${BASE_URL:-"http://$HOST:$PORT"} -integrationFolder=../../e2e -testFiles=*-suite/*spec.ts + +declare -A env=( + [BASE_URL]=${BASE_URL:-"http://$HOST:$PORT"} + [SLOWMO]=0 +) + +testFilesForSingleSuite="*.spec.ts" + +declare -A cypressConfig=( + [integrationFolder]=../../e2e + [screenshotsFolder]=../../e2e/"${args[0]}"/screenshots + [videosFolder]=../../e2e/"${args[0]}"/videos + [fileServerFolder]=./cypress + [testFiles]=*-suite/*spec.ts + [defaultCommandTimeout]=30000 + [viewportWidth]=1920 + [viewportHeight]=1080 + [trashAssetsBeforeRuns]=false + [videoUploadOnPasses]=false +) + cd packages/grafana-e2e -case "$1" in +case "$1" in "debug") echo -e "Debug mode" - SLOWMO=1 + env[SLOWMO]=1 PARAMS="--no-exit" ;; "dev") echo "Dev mode" CMD="open" ;; + "benchmark") + echo "Benchmark" + PARAMS="--headed" + CMD="start-benchmark" + env[BENCHMARK_PLUGIN_ENABLED]=true + env[BENCHMARK_PLUGIN_RESULTS_FOLDER]=../../e2e/benchmarks/"${args[1]}"/results + cypressConfig[video]=false + cypressConfig[integrationFolder]=../../e2e/benchmarks/"${args[1]}" + cypressConfig[screenshotsFolder]=../../e2e/benchmarks/"${args[1]}"/screenshots + cypressConfig[testFiles]=$testFilesForSingleSuite + ;; "") ;; *) - integrationFolder=../../e2e/"${args[0]}" - testFiles="*.spec.ts" + cypressConfig[integrationFolder]=../../e2e/"${args[0]}" + cypressConfig[testFiles]=$testFilesForSingleSuite ;; esac -yarn $CMD --env BASE_URL=$URL,SLOWMO=$SLOWMO \ - --config defaultCommandTimeout=30000,testFiles=$testFiles,integrationFolder=$integrationFolder,screenshotsFolder=../../e2e/"${args[0]}"/screenshots,videosFolder=../../e2e/"${args[0]}"/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false,videoUploadOnPasses=false \ +function join () { + local -n map=$1 + local delimiter="," + + local res="" + + for key in "${!map[@]}" + do + value=${map[$key]} + if [ -z "${res}" ]; then + res=$key=$value + else + res=$res$delimiter$key=$value + fi + done + + echo "$res" +} + + +yarn $CMD --env "$(join env)" \ + --config "$(join cypressConfig)" \ $PARAMS diff --git a/e2e/start-server b/e2e/start-server index 57a79b8645f..8d25be26d1e 100755 --- a/e2e/start-server +++ b/e2e/start-server @@ -36,6 +36,7 @@ else mkdir $PROV_DIR/datasources mkdir $PROV_DIR/dashboards + cp ./e2e/custom.ini $RUNDIR/conf/custom.ini cp ./conf/defaults.ini $RUNDIR/conf/defaults.ini fi diff --git a/package.json b/package.json index f0cec27d92b..63891ce2606 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "e2e": "./e2e/start-and-run-suite", "e2e:debug": "./e2e/start-and-run-suite debug", "e2e:dev": "./e2e/start-and-run-suite dev", + "e2e:benchmark:live": "./e2e/start-and-run-suite benchmark live", "test": "jest --notify --watch", "test:accessibility-report": "./scripts/generate-a11y-report.sh", "lint": "yarn run lint:ts && yarn run lint:sass", diff --git a/packages/grafana-e2e/cypress/plugins/benchmark/CDPDataCollector.ts b/packages/grafana-e2e/cypress/plugins/benchmark/CDPDataCollector.ts new file mode 100644 index 00000000000..b3f59028d06 --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/benchmark/CDPDataCollector.ts @@ -0,0 +1,136 @@ +import CDP from 'chrome-remote-interface'; +import Tracelib, { TraceEvent } from 'tracelib'; + +import { countBy, mean } from 'lodash'; +import ProtocolProxyApi from 'devtools-protocol/types/protocol-proxy-api'; +import { CollectedData, DataCollector, DataCollectorName } from './DataCollector'; + +type CDPDataCollectorDeps = { + port: number; +}; + +export class CDPDataCollector implements DataCollector { + private tracingCategories: string[]; + + private state: { + client?: CDP.Client; + tracingPromise?: Promise; + traceEvents: TraceEvent[]; + }; + + constructor(private deps: CDPDataCollectorDeps) { + this.state = this.getDefaultState(); + this.tracingCategories = [ + 'disabled-by-default-v8.cpu_profile', + 'disabled-by-default-v8.cpu_profiler', + 'disabled-by-default-v8.cpu_profiler.hires', + 'disabled-by-default-devtools.timeline.frame', + 'disabled-by-default-devtools.timeline', + 'disabled-by-default-devtools.timeline.inputs', + 'disabled-by-default-devtools.timeline.stack', + 'disabled-by-default-devtools.timeline.invalidationTracking', + 'disabled-by-default-layout_shift.debug', + 'disabled-by-default-cc.debug.scheduler.frames', + 'disabled-by-default-blink.debug.display_lock', + ]; + } + + getName = () => DataCollectorName.CDP; + + private resetState = async () => { + if (this.state.client) { + await this.state.client.close(); + } + this.state = this.getDefaultState(); + }; + + private getDefaultState = () => ({ + traceEvents: [], + }); + + // workaround for type declaration issues in cdp lib + private asApis = ( + client: CDP.Client + ): { + Profiler: ProtocolProxyApi.ProfilerApi; + Page: ProtocolProxyApi.PageApi; + Tracing: ProtocolProxyApi.TracingApi; + } => client; + + private getClientApis = async () => this.asApis(await this.getClient()); + + private getClient = async () => { + if (this.state.client) { + return this.state.client; + } + + const client = await CDP({ port: this.deps.port }); + + const { Profiler, Page } = this.asApis(client); + await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]); + + this.state.client = client; + + return client; + }; + + start: DataCollector['start'] = async ({ id }) => { + if (this.state.tracingPromise) { + throw new Error(`collection in progress - can't start another one! ${id}`); + } + + const { Tracing, Profiler } = await this.getClientApis(); + + await Promise.all([ + Tracing.start({ + bufferUsageReportingInterval: 1000, + traceConfig: { + includedCategories: this.tracingCategories, + }, + }), + Profiler.start(), + ]); + + Tracing.on('dataCollected', ({ value: events }) => { + this.state.traceEvents.push(...events); + }); + + let resolveFn: (data: CollectedData) => void; + this.state.tracingPromise = new Promise((resolve) => { + resolveFn = resolve; + }); + Tracing.on('tracingComplete', ({ dataLossOccurred }) => { + const t = new Tracelib(this.state.traceEvents); + + const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name); + + const fps = t.getFPS(); + + resolveFn({ + eventCounts, + fps: mean(fps.values), + tracingDataLoss: dataLossOccurred ? 1 : 0, + warnings: t.getWarningCounts(), + }); + }); + }; + + stop: DataCollector['stop'] = async (req) => { + if (!this.state.tracingPromise) { + throw new Error(`collection was never started - there is nothing to stop!`); + } + + const { Tracing, Profiler } = await this.getClientApis(); + + // TODO: capture profiler data + const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]); + + await this.resetState(); + + return traceData; + }; + + close: DataCollector['close'] = async () => { + await this.resetState(); + }; +} diff --git a/packages/grafana-e2e/cypress/plugins/benchmark/DataCollector.ts b/packages/grafana-e2e/cypress/plugins/benchmark/DataCollector.ts new file mode 100644 index 00000000000..bfff10312e2 --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/benchmark/DataCollector.ts @@ -0,0 +1,14 @@ +export type CollectedData = Record; + +export enum DataCollectorName { + CDP = 'CDP', +} + +type DataCollectorRequest = { id: string }; + +export type DataCollector = { + start: (input: DataCollectorRequest) => Promise; + stop: (input: DataCollectorRequest) => Promise; + getName: () => DataCollectorName; + close: () => Promise; +}; diff --git a/packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts b/packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts new file mode 100644 index 00000000000..8372309f50f --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/benchmark/formatting.ts @@ -0,0 +1,138 @@ +import { CollectedData, DataCollectorName } from './DataCollector'; +import { fromPairs } from 'lodash'; + +type Stats = { + sum: number; + min: number; + max: number; + count: number; + avg: number; + time: number; +}; + +export enum MeasurementName { + DataRenderDelay = 'DataRenderDelay', +} + +type LivePerformanceAppStats = Record; + +const isLivePerformanceAppStats = (data: CollectedData[]): data is LivePerformanceAppStats[] => + data.some((st) => { + const stat = st?.[MeasurementName.DataRenderDelay]; + return Array.isArray(stat) && Boolean(stat?.length); + }); + +type FormattedStats = { + total: { + count: number[]; + avg: number[]; + }; + lastInterval: { + avg: number[]; + min: number[]; + max: number[]; + count: number[]; + }; +}; + +export const formatAppStats = (allStats: CollectedData[]) => { + if (!isLivePerformanceAppStats(allStats)) { + return {}; + } + + const names = Object.keys(MeasurementName) as MeasurementName[]; + + return fromPairs( + names.map((name) => { + const statsForMeasurement = allStats.map((s) => s[name]); + const res: FormattedStats = { + total: { + count: [], + avg: [], + }, + lastInterval: { + avg: [], + min: [], + max: [], + count: [], + }, + }; + + statsForMeasurement.forEach((s) => { + const total = s.reduce( + (prev, next) => { + prev.count += next.count; + prev.avg += next.avg; + return prev; + }, + { count: 0, avg: 0 } + ); + res.total.count.push(Math.round(total.count)); + res.total.avg.push(Math.round(total.avg / s.length)); + + const lastInterval = s[s.length - 1]; + + res.lastInterval.avg.push(Math.round(lastInterval?.avg)); + res.lastInterval.min.push(Math.round(lastInterval?.min)); + res.lastInterval.max.push(Math.round(lastInterval?.max)); + res.lastInterval.count.push(Math.round(lastInterval?.count)); + }); + + return [name, res]; + }) + ); +}; + +type CDPData = { + eventCounts: Record; + fps: number; + tracingDataLoss: number; + warnings: Record; +}; + +const isCDPData = (data: any[]): data is CDPData[] => data.every((d) => typeof d.eventCounts === 'object'); + +type FormattedCDPData = { + minorGC: number[]; + majorGC: number[]; + droppedFrames: number[]; + fps: number[]; + tracingDataLossOccurred: boolean; + longTaskWarnings: number[]; +}; + +const emptyFormattedCDPData = (): FormattedCDPData => ({ + minorGC: [], + majorGC: [], + droppedFrames: [], + fps: [], + tracingDataLossOccurred: false, + longTaskWarnings: [], +}); + +const formatCDPData = (data: any): FormattedCDPData => { + if (!isCDPData(data)) { + return emptyFormattedCDPData(); + } + + return data.reduce((acc, next) => { + acc.majorGC.push((next.eventCounts.MajorGC as number) ?? 0); + acc.minorGC.push((next.eventCounts.MinorGC as number) ?? 0); + acc.fps.push(Math.round(next.fps) ?? 0); + acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss); + acc.droppedFrames.push((next.eventCounts.DroppedFrame as number) ?? 0); + acc.longTaskWarnings.push((next.warnings.LongTask as number) ?? 0); + return acc; + }, emptyFormattedCDPData()); +}; + +export const formatResults = ( + results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> +): CollectedData => { + return { + ...formatAppStats(results.map(({ appStats }) => appStats)), + ...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])), + + __raw: results, + }; +}; diff --git a/packages/grafana-e2e/cypress/plugins/benchmark/index.ts b/packages/grafana-e2e/cypress/plugins/benchmark/index.ts new file mode 100644 index 00000000000..d822b40dc67 --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/benchmark/index.ts @@ -0,0 +1,87 @@ +import { CollectedData, DataCollector } from './DataCollector'; +import { CDPDataCollector } from './CDPDataCollector'; +import { fromPairs } from 'lodash'; +import fs from 'fs'; +import { formatResults } from './formatting'; +const remoteDebuggingPortOptionPrefix = '--remote-debugging-port='; + +const getOrAddRemoteDebuggingPort = (args: string[]) => { + const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix)); + + if (existing) { + return Number(existing.substring(remoteDebuggingPortOptionPrefix.length)); + } + + const port = 40000 + Math.round(Math.random() * 25000); + args.push(`${remoteDebuggingPortOptionPrefix}${port}`); + return port; +}; + +let collectors: DataCollector[] = []; +let results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> = []; + +const startBenchmarking = async ({ testName }: { testName: string }) => { + await Promise.all(collectors.map((coll) => coll.start({ id: testName }))); + + return true; +}; + +const stopBenchmarking = async ({ testName, appStats }: { testName: string; appStats: CollectedData }) => { + const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })])); + + results.push({ + collectorsData: fromPairs(data), + appStats: appStats, + }); + + return true; +}; +const afterRun = async () => { + await Promise.all(collectors.map((coll) => coll.close())); + collectors = []; + results = []; +}; + +const afterSpec = (resultsFolder: string) => async (spec: { name: string }) => { + fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2)); + + results = []; +}; + +export const initialize: Cypress.PluginConfig = (on, config) => { + const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER']; + + if (!fs.existsSync(resultsFolder)) { + fs.mkdirSync(resultsFolder, { recursive: true }); + console.log(`Created folder for benchmark results ${resultsFolder}`); + } + + on('before:browser:launch', async (browser, options) => { + if (browser.family !== 'chromium' || browser.name === 'electron') { + throw new Error('benchmarking plugin requires chrome'); + } + + const { args } = options; + + const port = getOrAddRemoteDebuggingPort(args); + collectors.push(new CDPDataCollector({ port })); + + args.push('--start-fullscreen'); + + console.log( + `initialized benchmarking plugin with ${collectors.length} collectors: ${collectors + .map((col) => col.getName()) + .join(', ')}` + ); + + return options; + }); + + on('task', { + startBenchmarking, + stopBenchmarking, + }); + + on('after:run', afterRun); + on('after:spec', afterSpec(resultsFolder)); +}; diff --git a/packages/grafana-e2e/cypress/plugins/benchmark/tracelib.d.ts b/packages/grafana-e2e/cypress/plugins/benchmark/tracelib.d.ts new file mode 100644 index 00000000000..2ba83b35594 --- /dev/null +++ b/packages/grafana-e2e/cypress/plugins/benchmark/tracelib.d.ts @@ -0,0 +1,15 @@ +type TraceEvent = { + name: string; +}; + +declare class Tracelib { + constructor(private events: TraceEvent[]) {} + + getFPS: () => { times: number[]; values: number[] }; + getWarningCounts: () => Record; +} +declare module 'tracelib' { + export = Tracelib; + + export { TraceEvent }; +} diff --git a/packages/grafana-e2e/cypress/plugins/index.js b/packages/grafana-e2e/cypress/plugins/index.js index 8b0f5dfdb46..d9c4eebba76 100644 --- a/packages/grafana-e2e/cypress/plugins/index.js +++ b/packages/grafana-e2e/cypress/plugins/index.js @@ -5,8 +5,13 @@ const compareScreenshots = require('./compareScreenshots'); const extendConfig = require('./extendConfig'); const readProvisions = require('./readProvisions'); const typescriptPreprocessor = require('./typescriptPreprocessor'); +const benchmarkPlugin = require('./benchmark'); module.exports = (on, config) => { + if (config.env['BENCHMARK_PLUGIN_ENABLED'] === true) { + benchmarkPlugin.initialize(on, config); + } + on('file:preprocessor', typescriptPreprocessor); on('task', { compareScreenshots, readProvisions }); on('task', { diff --git a/packages/grafana-e2e/cypress/support/commands.ts b/packages/grafana-e2e/cypress/support/commands.ts index 48e933bf34c..ce386d11d8f 100644 --- a/packages/grafana-e2e/cypress/support/commands.ts +++ b/packages/grafana-e2e/cypress/support/commands.ts @@ -31,3 +31,11 @@ Cypress.Commands.add('getJSONFilesFromDir', (dirPath: string) => { relativePath: dirPath, }); }); + +Cypress.Commands.add('startBenchmarking', (testName: string) => { + return cy.task('startBenchmarking', { testName }); +}); + +Cypress.Commands.add('stopBenchmarking', (testName: string, appStats: Record) => { + return cy.task('stopBenchmarking', { testName, appStats }); +}); diff --git a/packages/grafana-e2e/cypress/support/index.d.ts b/packages/grafana-e2e/cypress/support/index.d.ts index 224b21bd339..3f8d1975936 100644 --- a/packages/grafana-e2e/cypress/support/index.d.ts +++ b/packages/grafana-e2e/cypress/support/index.d.ts @@ -6,5 +6,7 @@ declare namespace Cypress { logToConsole(message: string, optional?: any): void; readProvisions(filePaths: string[]): Chainable; getJSONFilesFromDir(dirPath: string): Chainable; + startBenchmarking(testName: string): void; + stopBenchmarking(testName: string, appStats: Record): void; } } diff --git a/packages/grafana-e2e/cypress/tsconfig.json b/packages/grafana-e2e/cypress/tsconfig.json index c7e1a52579e..b3f735c508e 100644 --- a/packages/grafana-e2e/cypress/tsconfig.json +++ b/packages/grafana-e2e/cypress/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "declaration": false, "module": "commonjs", - "types": ["cypress", "cypress-file-upload"] + "types": ["cypress", "cypress-file-upload", "node"] }, "extends": "@grafana/tsconfig", "include": ["**/*.ts"] diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index 1162c1c8c4f..ad965d1c0c8 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -26,12 +26,15 @@ "docsExtract": "mkdir -p ../../reports/docs && api-extractor run 2>&1 | tee ../../reports/docs/$(basename $(pwd)).log", "open": "cypress open", "start": "cypress run --browser=chrome", + "start-benchmark": "CYPRESS_NO_COMMAND_LOG=1 yarn start", "test": "pushd test && node ../dist/bin/grafana-e2e.js run", "typecheck": "tsc --noEmit" }, "devDependencies": { "@rollup/plugin-commonjs": "21.0.1", "@rollup/plugin-node-resolve": "13.1.3", + "@types/chrome-remote-interface": "0.31.4", + "@types/lodash": "4.14.149", "@types/node": "16.11.19", "@types/uuid": "8.3.4", "rollup": "2.63.0", @@ -50,13 +53,17 @@ "@mochajs/json-file-reporter": "^1.2.0", "babel-loader": "8.2.3", "blink-diff": "1.0.13", + "chrome-remote-interface": "0.31.1", "commander": "8.3.0", "cypress": "9.2.0", "cypress-file-upload": "5.0.8", + "devtools-protocol": "0.0.927104", "execa": "5.1.1", + "lodash": "4.17.21", "mocha": "9.1.3", "resolve-as-bin": "2.1.0", "rimraf": "3.0.2", + "tracelib": "1.0.1", "ts-loader": "6.2.1", "tslib": "2.3.1", "typescript": "4.5.4", diff --git a/packages/grafana-e2e/src/flows/importDashboard.ts b/packages/grafana-e2e/src/flows/importDashboard.ts index 84631082655..c054dd4ad3a 100644 --- a/packages/grafana-e2e/src/flows/importDashboard.ts +++ b/packages/grafana-e2e/src/flows/importDashboard.ts @@ -13,8 +13,9 @@ export type Dashboard = { title: string; panels: Panel[]; uid: string; [key: str * Smoke test a particular dashboard by quickly importing a json file and validate that all the panels finish loading * @param dashboardToImport a sample dashboard * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading + * @param skipPanelValidation skip panel validation */ -export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number) => { +export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: number, skipPanelValidation?: boolean) => { e2e().visit(fromBaseUrl('/dashboard/import')); // Note: normally we'd use 'click' and then 'type' here, but the json object is so big that using 'val' is much faster @@ -45,21 +46,25 @@ export const importDashboard = (dashboardToImport: Dashboard, queryTimeout?: num expect(dashboardToImport.uid).to.equal(uid); }); - dashboardToImport.panels.forEach((panel) => { - // Look at the json data - e2e.components.Panels.Panel.title(panel.title).should('be.visible').click(); - e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click(); - e2e.components.Tab.title('JSON').should('be.visible').click(); - e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true }); - e2e.components.Select.option().should('be.visible').contains('Data').click(); + if (!skipPanelValidation) { + dashboardToImport.panels.forEach((panel) => { + // Look at the json data + e2e.components.Panels.Panel.title(panel.title).should('be.visible').click(); + e2e.components.Panels.Panel.headerItems('Inspect').should('be.visible').click(); + e2e.components.Tab.title('JSON').should('be.visible').click(); + e2e.components.PanelInspector.Json.content().should('be.visible').contains('Panel JSON').click({ force: true }); + e2e.components.Select.option().should('be.visible').contains('Data').click(); - // ensures that panel has loaded without knowingly hitting an error - // note: this does not prove that data came back as we expected it, - // it could get `state: Done` for no data for example - // but it ensures we didn't hit a 401 or 500 or something like that - e2e.components.CodeEditor.container().should('be.visible').contains('"state": "Done"'); + // ensures that panel has loaded without knowingly hitting an error + // note: this does not prove that data came back as we expected it, + // it could get `state: Done` for no data for example + // but it ensures we didn't hit a 401 or 500 or something like that + e2e.components.CodeEditor.container() + .should('be.visible') + .contains(/"state": "(Done|Streaming)"/); - // need to close panel - e2e.components.Drawer.General.close().click(); - }); + // need to close panel + e2e.components.Drawer.General.close().click(); + }); + } }; diff --git a/packages/grafana-e2e/src/flows/importDashboards.ts b/packages/grafana-e2e/src/flows/importDashboards.ts index 6a4c2a5de85..24b7bd356f8 100644 --- a/packages/grafana-e2e/src/flows/importDashboards.ts +++ b/packages/grafana-e2e/src/flows/importDashboards.ts @@ -7,13 +7,14 @@ import { e2e } from '../index'; * @param dirPath the relative path to a directory which contains json files representing dashboards, * for example if your dashboards live in `cypress/testDashboards` you can pass `/testDashboards` * @param queryTimeout a number of ms to wait for the imported dashboard to finish loading + * @param skipPanelValidation skips panel validation */ -export const importDashboards = async (dirPath: string, queryTimeout?: number) => { +export const importDashboards = async (dirPath: string, queryTimeout?: number, skipPanelValidation?: boolean) => { e2e() .getJSONFilesFromDir(dirPath) .then((jsonFiles: Dashboard[]) => { jsonFiles.forEach((file) => { - importDashboard(file, queryTimeout || 6000); + importDashboard(file, queryTimeout || 6000, skipPanelValidation); }); }); }; diff --git a/packages/grafana-e2e/src/index.ts b/packages/grafana-e2e/src/index.ts index 87a0a6a8f25..b4c62c16352 100644 --- a/packages/grafana-e2e/src/index.ts +++ b/packages/grafana-e2e/src/index.ts @@ -4,6 +4,7 @@ * @packageDocumentation */ import { e2eScenario, ScenarioArguments } from './support/scenario'; +import { benchmark } from './support/benchmark'; import { getScenarioContext, setScenarioContext } from './support/scenarioContext'; import { e2eFactory } from './support'; import { E2ESelectors, Selectors, selectors } from '@grafana/e2e-selectors'; @@ -16,6 +17,7 @@ const e2eObject = { blobToBase64String: (blob: any) => Cypress.Blob.blobToBase64String(blob), imgSrcToBlob: (url: string) => Cypress.Blob.imgSrcToBlob(url), scenario: (args: ScenarioArguments) => e2eScenario(args), + benchmark, pages: e2eFactory({ selectors: selectors.pages }), typings, components: e2eFactory({ selectors: selectors.components }), diff --git a/packages/grafana-e2e/src/support/benchmark.ts b/packages/grafana-e2e/src/support/benchmark.ts new file mode 100644 index 00000000000..70dbfb45028 --- /dev/null +++ b/packages/grafana-e2e/src/support/benchmark.ts @@ -0,0 +1,81 @@ +import { e2e } from '../'; + +export interface BenchmarkArguments { + name: string; + dashboard: { + folder: string; + delayAfterOpening: number; + skipPanelValidation: boolean; + }; + repeat: number; + duration: number; + appStats?: { + startCollecting?: (window: Window) => void; + collect: (window: Window) => Record; + }; + skipScenario?: boolean; +} + +export const benchmark = ({ + name, + skipScenario = false, + repeat, + duration, + appStats, + dashboard, +}: BenchmarkArguments) => { + if (skipScenario) { + describe(name, () => { + it.skip(name, () => {}); + }); + } + + describe(name, () => { + before(() => { + e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD')); + }); + + beforeEach(() => { + e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation); + Cypress.Cookies.preserveOnce('grafana_session'); + }); + + afterEach(() => e2e.flows.revertAllChanges()); + after(() => { + e2e().clearCookies(); + }); + + Array(repeat) + .fill(0) + .map((_, i) => { + const testName = `${name}-${i}`; + return it(testName, () => { + e2e.flows.openDashboard(); + + e2e().wait(dashboard.delayAfterOpening); + + if (appStats) { + const startCollecting = appStats.startCollecting; + if (startCollecting) { + e2e() + .window() + .then((win) => startCollecting(win)); + } + + e2e().startBenchmarking(testName); + e2e().wait(duration); + + e2e() + .window() + .then((win) => { + e2e().stopBenchmarking(testName, appStats.collect(win)); + }); + } else { + e2e().startBenchmarking(testName); + e2e().wait(duration); + e2e().stopBenchmarking(testName, {}); + } + }); + }); + }); +}; diff --git a/yarn.lock b/yarn.lock index b434bd85f14..873efb23b96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3782,14 +3782,19 @@ __metadata: "@mochajs/json-file-reporter": ^1.2.0 "@rollup/plugin-commonjs": 21.0.1 "@rollup/plugin-node-resolve": 13.1.3 + "@types/chrome-remote-interface": 0.31.4 + "@types/lodash": 4.14.149 "@types/node": 16.11.19 "@types/uuid": 8.3.4 babel-loader: 8.2.3 blink-diff: 1.0.13 + chrome-remote-interface: 0.31.1 commander: 8.3.0 cypress: 9.2.0 cypress-file-upload: 5.0.8 + devtools-protocol: 0.0.927104 execa: 5.1.1 + lodash: 4.17.21 mocha: 9.1.3 resolve-as-bin: 2.1.0 rimraf: 3.0.2 @@ -3797,6 +3802,7 @@ __metadata: rollup-plugin-copy: 3.4.0 rollup-plugin-sourcemaps: 0.6.3 rollup-plugin-terser: 7.0.2 + tracelib: 1.0.1 ts-loader: 6.2.1 tslib: 2.3.1 typescript: 4.5.4 @@ -8946,6 +8952,15 @@ __metadata: languageName: node linkType: hard +"@types/chrome-remote-interface@npm:0.31.4": + version: 0.31.4 + resolution: "@types/chrome-remote-interface@npm:0.31.4" + dependencies: + devtools-protocol: 0.0.927104 + checksum: 91c6cf9c749adedc08458b772db12f4142172f36fe885dc741921d259bdb7ec47d64b9e294d793a6c36f0435c6855d6470754fd143711e0b2ff204878dd4f721 + languageName: node + linkType: hard + "@types/classnames@npm:2.3.0, @types/classnames@npm:^2.2.7": version: 2.3.0 resolution: "@types/classnames@npm:2.3.0" @@ -14078,6 +14093,18 @@ __metadata: languageName: node linkType: hard +"chrome-remote-interface@npm:0.31.1": + version: 0.31.1 + resolution: "chrome-remote-interface@npm:0.31.1" + dependencies: + commander: 2.11.x + ws: ^7.2.0 + bin: + chrome-remote-interface: bin/client.js + checksum: fbc7776abaa58b1d9b517ce93c479c0cb5da287df4a97908675f77cc261c145a772d2ba19d4d23845b7800eae1354008278b0a4e410ff419f641a3f43bbcef15 + languageName: node + linkType: hard + "chrome-trace-event@npm:^1.0.2": version: 1.0.3 resolution: "chrome-trace-event@npm:1.0.3" @@ -14510,6 +14537,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:2.11.x": + version: 2.11.0 + resolution: "commander@npm:2.11.0" + checksum: 0d0c622d129a801699b9bbf6fa518108c7e221e51ae12457119aec52f1142ab759b6cd3348ee253604e934639e200c8f0e1cf8342a2ba4b28b3565a7322ead14 + languageName: node + linkType: hard + "commander@npm:2.17.x": version: 2.17.1 resolution: "commander@npm:2.17.1" @@ -16642,6 +16676,13 @@ __metadata: languageName: node linkType: hard +"devtools-protocol@npm:0.0.927104": + version: 0.0.927104 + resolution: "devtools-protocol@npm:0.0.927104" + checksum: 13617e735f326b9822e64480060e59068434e937a6141ffe18353d0c7626be890e94d3118e7c262e6fd17eea3a4a2a18c221d0ff1eb31768ebcf0a0b95475d32 + languageName: node + linkType: hard + "dezalgo@npm:^1.0.0": version: 1.0.3 resolution: "dezalgo@npm:1.0.3" @@ -34063,6 +34104,13 @@ __metadata: languageName: node linkType: hard +"tracelib@npm:1.0.1": + version: 1.0.1 + resolution: "tracelib@npm:1.0.1" + checksum: b4b7899491b6a2279e297d3fffca60ae330992a18af5a9cc85436159b8e313f42bc54794d61f504619b271531feea74bcd5b99950a1bed3a51378c23c3fa2e4a + languageName: node + linkType: hard + "traverse@npm:^0.6.6": version: 0.6.6 resolution: "traverse@npm:0.6.6" @@ -36231,6 +36279,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:^7.2.0": + version: 7.5.6 + resolution: "ws@npm:7.5.6" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 0c2ffc9a539dd61dd2b00ff6cc5c98a3371e2521011fe23da4b3578bb7ac26cbdf7ca8a68e8e08023c122ae247013216dde2a20c908de415a6bcc87bdef68c87 + languageName: node + linkType: hard + "ws@npm:^8.1.0": version: 8.2.3 resolution: "ws@npm:8.2.3"