mirror of https://github.com/grafana/grafana
Live: performance tests e2e part (#43915)
* #41993: live perf tests e2e part * #41993: added bash upgrade instructions * #41993: remove custom feature toggle * #41993: fix typo in 'integrationFolder'pull/43180/head^2
Artur Wierzbicki
4 years ago
committed by
GitHub
parent
0c88b39162
commit
e01ac44cfa
@ -0,0 +1,399 @@ |
||||
{ |
||||
"annotations": { |
||||
"list": [ |
||||
{ |
||||
"builtIn": 1, |
||||
"datasource": "-- Grafana --", |
||||
"enable": true, |
||||
"hide": true, |
||||
"iconColor": "rgba(0, 211, 255, 1)", |
||||
"name": "Annotations & Alerts", |
||||
"target": { |
||||
"limit": 100, |
||||
"matchAny": false, |
||||
"tags": [], |
||||
"type": "dashboard" |
||||
}, |
||||
"type": "dashboard" |
||||
} |
||||
] |
||||
}, |
||||
"editable": true, |
||||
"fiscalYearStartMonth": 0, |
||||
"graphTooltip": 0, |
||||
"id": 82, |
||||
"links": [], |
||||
"liveNow": false, |
||||
"panels": [ |
||||
{ |
||||
"datasource": { |
||||
"type": "grafana", |
||||
"uid": "grafana" |
||||
}, |
||||
"fieldConfig": { |
||||
"defaults": { |
||||
"color": { |
||||
"mode": "palette-classic" |
||||
}, |
||||
"custom": { |
||||
"axisLabel": "", |
||||
"axisPlacement": "auto", |
||||
"barAlignment": 0, |
||||
"drawStyle": "line", |
||||
"fillOpacity": 0, |
||||
"gradientMode": "none", |
||||
"hideFrom": { |
||||
"legend": false, |
||||
"tooltip": false, |
||||
"viz": false |
||||
}, |
||||
"lineInterpolation": "linear", |
||||
"lineWidth": 1, |
||||
"pointSize": 5, |
||||
"scaleDistribution": { |
||||
"type": "linear" |
||||
}, |
||||
"showPoints": "auto", |
||||
"spanNulls": false, |
||||
"stacking": { |
||||
"group": "A", |
||||
"mode": "none" |
||||
}, |
||||
"thresholdsStyle": { |
||||
"mode": "off" |
||||
} |
||||
}, |
||||
"mappings": [], |
||||
"thresholds": { |
||||
"mode": "absolute", |
||||
"steps": [ |
||||
{ |
||||
"color": "green", |
||||
"value": null |
||||
}, |
||||
{ |
||||
"color": "red", |
||||
"value": 80 |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"overrides": [] |
||||
}, |
||||
"gridPos": { |
||||
"h": 8, |
||||
"w": 12, |
||||
"x": 0, |
||||
"y": 0 |
||||
}, |
||||
"id": 10, |
||||
"maxDataPoints": 1500, |
||||
"options": { |
||||
"legend": { |
||||
"calcs": [], |
||||
"displayMode": "list", |
||||
"placement": "bottom" |
||||
}, |
||||
"tooltip": { |
||||
"mode": "single" |
||||
} |
||||
}, |
||||
"targets": [ |
||||
{ |
||||
"channel": "plugin/testdata/random-20Hz-stream-5", |
||||
"datasource": { |
||||
"type": "datasource", |
||||
"uid": "grafana" |
||||
}, |
||||
"queryType": "measurements", |
||||
"refId": "A" |
||||
} |
||||
], |
||||
"title": "5", |
||||
"type": "timeseries" |
||||
}, |
||||
{ |
||||
"datasource": { |
||||
"type": "grafana", |
||||
"uid": "grafana" |
||||
}, |
||||
"fieldConfig": { |
||||
"defaults": { |
||||
"color": { |
||||
"mode": "palette-classic" |
||||
}, |
||||
"custom": { |
||||
"axisLabel": "", |
||||
"axisPlacement": "auto", |
||||
"barAlignment": 0, |
||||
"drawStyle": "line", |
||||
"fillOpacity": 0, |
||||
"gradientMode": "none", |
||||
"hideFrom": { |
||||
"legend": false, |
||||
"tooltip": false, |
||||
"viz": false |
||||
}, |
||||
"lineInterpolation": "linear", |
||||
"lineWidth": 1, |
||||
"pointSize": 5, |
||||
"scaleDistribution": { |
||||
"type": "linear" |
||||
}, |
||||
"showPoints": "auto", |
||||
"spanNulls": false, |
||||
"stacking": { |
||||
"group": "A", |
||||
"mode": "none" |
||||
}, |
||||
"thresholdsStyle": { |
||||
"mode": "off" |
||||
} |
||||
}, |
||||
"mappings": [], |
||||
"thresholds": { |
||||
"mode": "absolute", |
||||
"steps": [ |
||||
{ |
||||
"color": "green", |
||||
"value": null |
||||
}, |
||||
{ |
||||
"color": "red", |
||||
"value": 80 |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"overrides": [] |
||||
}, |
||||
"gridPos": { |
||||
"h": 8, |
||||
"w": 12, |
||||
"x": 12, |
||||
"y": 0 |
||||
}, |
||||
"id": 6, |
||||
"maxDataPoints": 1500, |
||||
"options": { |
||||
"legend": { |
||||
"calcs": [], |
||||
"displayMode": "list", |
||||
"placement": "bottom" |
||||
}, |
||||
"tooltip": { |
||||
"mode": "single" |
||||
} |
||||
}, |
||||
"targets": [ |
||||
{ |
||||
"channel": "plugin/testdata/random-20Hz-stream-2", |
||||
"datasource": { |
||||
"type": "datasource", |
||||
"uid": "grafana" |
||||
}, |
||||
"queryType": "measurements", |
||||
"refId": "A" |
||||
} |
||||
], |
||||
"title": "2", |
||||
"type": "timeseries" |
||||
}, |
||||
{ |
||||
"datasource": { |
||||
"type": "grafana", |
||||
"uid": "grafana" |
||||
}, |
||||
"fieldConfig": { |
||||
"defaults": { |
||||
"color": { |
||||
"mode": "palette-classic" |
||||
}, |
||||
"custom": { |
||||
"axisLabel": "", |
||||
"axisPlacement": "auto", |
||||
"barAlignment": 0, |
||||
"drawStyle": "line", |
||||
"fillOpacity": 0, |
||||
"gradientMode": "none", |
||||
"hideFrom": { |
||||
"legend": false, |
||||
"tooltip": false, |
||||
"viz": false |
||||
}, |
||||
"lineInterpolation": "linear", |
||||
"lineWidth": 1, |
||||
"pointSize": 5, |
||||
"scaleDistribution": { |
||||
"type": "linear" |
||||
}, |
||||
"showPoints": "auto", |
||||
"spanNulls": false, |
||||
"stacking": { |
||||
"group": "A", |
||||
"mode": "none" |
||||
}, |
||||
"thresholdsStyle": { |
||||
"mode": "off" |
||||
} |
||||
}, |
||||
"mappings": [], |
||||
"thresholds": { |
||||
"mode": "absolute", |
||||
"steps": [ |
||||
{ |
||||
"color": "green", |
||||
"value": null |
||||
}, |
||||
{ |
||||
"color": "red", |
||||
"value": 80 |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"overrides": [] |
||||
}, |
||||
"gridPos": { |
||||
"h": 8, |
||||
"w": 12, |
||||
"x": 0, |
||||
"y": 8 |
||||
}, |
||||
"id": 8, |
||||
"maxDataPoints": 1500, |
||||
"options": { |
||||
"legend": { |
||||
"calcs": [], |
||||
"displayMode": "list", |
||||
"placement": "bottom" |
||||
}, |
||||
"tooltip": { |
||||
"mode": "single" |
||||
} |
||||
}, |
||||
"targets": [ |
||||
{ |
||||
"channel": "plugin/testdata/random-20Hz-stream-4", |
||||
"datasource": { |
||||
"type": "datasource", |
||||
"uid": "grafana" |
||||
}, |
||||
"queryType": "measurements", |
||||
"refId": "A" |
||||
} |
||||
], |
||||
"title": "4", |
||||
"type": "timeseries" |
||||
}, |
||||
{ |
||||
"datasource": { |
||||
"type": "grafana", |
||||
"uid": "grafana" |
||||
}, |
||||
"fieldConfig": { |
||||
"defaults": { |
||||
"color": { |
||||
"mode": "palette-classic" |
||||
}, |
||||
"custom": { |
||||
"axisLabel": "", |
||||
"axisPlacement": "auto", |
||||
"barAlignment": 0, |
||||
"drawStyle": "line", |
||||
"fillOpacity": 0, |
||||
"gradientMode": "none", |
||||
"hideFrom": { |
||||
"legend": false, |
||||
"tooltip": false, |
||||
"viz": false |
||||
}, |
||||
"lineInterpolation": "linear", |
||||
"lineWidth": 3, |
||||
"pointSize": 5, |
||||
"scaleDistribution": { |
||||
"type": "linear" |
||||
}, |
||||
"showPoints": "auto", |
||||
"spanNulls": false, |
||||
"stacking": { |
||||
"group": "A", |
||||
"mode": "none" |
||||
}, |
||||
"thresholdsStyle": { |
||||
"mode": "off" |
||||
} |
||||
}, |
||||
"mappings": [], |
||||
"thresholds": { |
||||
"mode": "absolute", |
||||
"steps": [ |
||||
{ |
||||
"color": "green", |
||||
"value": null |
||||
}, |
||||
{ |
||||
"color": "red", |
||||
"value": 80 |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"overrides": [] |
||||
}, |
||||
"gridPos": { |
||||
"h": 9, |
||||
"w": 12, |
||||
"x": 12, |
||||
"y": 8 |
||||
}, |
||||
"id": 2, |
||||
"maxDataPoints": 1500, |
||||
"options": { |
||||
"legend": { |
||||
"calcs": [], |
||||
"displayMode": "list", |
||||
"placement": "bottom" |
||||
}, |
||||
"tooltip": { |
||||
"mode": "none" |
||||
} |
||||
}, |
||||
"targets": [ |
||||
{ |
||||
"buffer": 60000, |
||||
"channel": "plugin/testdata/random-20Hz-stream", |
||||
"datasource": { |
||||
"type": "datasource", |
||||
"uid": "grafana" |
||||
}, |
||||
"filter": { |
||||
"fields": [] |
||||
}, |
||||
"queryType": "measurements", |
||||
"refId": "A" |
||||
} |
||||
], |
||||
"title": "1", |
||||
"transparent": true, |
||||
"type": "timeseries" |
||||
} |
||||
], |
||||
"refresh": "", |
||||
"schemaVersion": 33, |
||||
"style": "dark", |
||||
"tags": [], |
||||
"templating": { |
||||
"list": [] |
||||
}, |
||||
"time": { |
||||
"from": "now-60s", |
||||
"to": "now" |
||||
}, |
||||
"timepicker": {}, |
||||
"timezone": "", |
||||
"title": "Grafana Live performance benchmarking", |
||||
"uid": "S4M5r9cnk", |
||||
"version": 9, |
||||
"weekStart": "" |
||||
} |
||||
@ -0,0 +1,41 @@ |
||||
import { e2e } from '@grafana/e2e'; |
||||
|
||||
type WithGrafanaRuntime<T> = T & { |
||||
grafanaRuntime: { |
||||
livePerformance: { |
||||
start: () => void; |
||||
getStats: () => Record<string, unknown>; |
||||
}; |
||||
}; |
||||
}; |
||||
|
||||
const hasGrafanaRuntime = <T>(obj: T): obj is WithGrafanaRuntime<T> => { |
||||
return typeof (obj as any)?.grafanaRuntime === 'object'; |
||||
}; |
||||
|
||||
e2e.benchmark({ |
||||
name: 'Live performance benchmarking - 4x20hz panels', |
||||
dashboard: { |
||||
folder: '/dashboards/live', |
||||
delayAfterOpening: 1000, |
||||
skipPanelValidation: true, |
||||
}, |
||||
repeat: 10, |
||||
duration: 120000, |
||||
appStats: { |
||||
startCollecting: (window) => { |
||||
if (!hasGrafanaRuntime(window)) { |
||||
return; |
||||
} |
||||
|
||||
return window.grafanaRuntime.livePerformance.start(); |
||||
}, |
||||
collect: (window) => { |
||||
if (!hasGrafanaRuntime(window)) { |
||||
return {}; |
||||
} |
||||
|
||||
return window.grafanaRuntime.livePerformance.getStats() ?? {}; |
||||
}, |
||||
}, |
||||
}); |
||||
@ -0,0 +1,5 @@ |
||||
{ |
||||
"extends": "../../tsconfig.json", |
||||
"include": ["**/*.ts", "../../packages/grafana-e2e/cypress/support/index.d.ts"], |
||||
"resolveJsonModule": true |
||||
} |
||||
@ -0,0 +1,2 @@ |
||||
[feature_toggles] |
||||
enable = |
||||
@ -0,0 +1,136 @@ |
||||
import CDP from 'chrome-remote-interface'; |
||||
import Tracelib, { TraceEvent } from 'tracelib'; |
||||
|
||||
import { countBy, mean } from 'lodash'; |
||||
import ProtocolProxyApi from 'devtools-protocol/types/protocol-proxy-api'; |
||||
import { CollectedData, DataCollector, DataCollectorName } from './DataCollector'; |
||||
|
||||
type CDPDataCollectorDeps = { |
||||
port: number; |
||||
}; |
||||
|
||||
export class CDPDataCollector implements DataCollector { |
||||
private tracingCategories: string[]; |
||||
|
||||
private state: { |
||||
client?: CDP.Client; |
||||
tracingPromise?: Promise<CollectedData>; |
||||
traceEvents: TraceEvent[]; |
||||
}; |
||||
|
||||
constructor(private deps: CDPDataCollectorDeps) { |
||||
this.state = this.getDefaultState(); |
||||
this.tracingCategories = [ |
||||
'disabled-by-default-v8.cpu_profile', |
||||
'disabled-by-default-v8.cpu_profiler', |
||||
'disabled-by-default-v8.cpu_profiler.hires', |
||||
'disabled-by-default-devtools.timeline.frame', |
||||
'disabled-by-default-devtools.timeline', |
||||
'disabled-by-default-devtools.timeline.inputs', |
||||
'disabled-by-default-devtools.timeline.stack', |
||||
'disabled-by-default-devtools.timeline.invalidationTracking', |
||||
'disabled-by-default-layout_shift.debug', |
||||
'disabled-by-default-cc.debug.scheduler.frames', |
||||
'disabled-by-default-blink.debug.display_lock', |
||||
]; |
||||
} |
||||
|
||||
getName = () => DataCollectorName.CDP; |
||||
|
||||
private resetState = async () => { |
||||
if (this.state.client) { |
||||
await this.state.client.close(); |
||||
} |
||||
this.state = this.getDefaultState(); |
||||
}; |
||||
|
||||
private getDefaultState = () => ({ |
||||
traceEvents: [], |
||||
}); |
||||
|
||||
// workaround for type declaration issues in cdp lib
|
||||
private asApis = ( |
||||
client: CDP.Client |
||||
): { |
||||
Profiler: ProtocolProxyApi.ProfilerApi; |
||||
Page: ProtocolProxyApi.PageApi; |
||||
Tracing: ProtocolProxyApi.TracingApi; |
||||
} => client; |
||||
|
||||
private getClientApis = async () => this.asApis(await this.getClient()); |
||||
|
||||
private getClient = async () => { |
||||
if (this.state.client) { |
||||
return this.state.client; |
||||
} |
||||
|
||||
const client = await CDP({ port: this.deps.port }); |
||||
|
||||
const { Profiler, Page } = this.asApis(client); |
||||
await Promise.all([Page.enable(), Profiler.enable(), Profiler.setSamplingInterval({ interval: 100 })]); |
||||
|
||||
this.state.client = client; |
||||
|
||||
return client; |
||||
}; |
||||
|
||||
start: DataCollector['start'] = async ({ id }) => { |
||||
if (this.state.tracingPromise) { |
||||
throw new Error(`collection in progress - can't start another one! ${id}`); |
||||
} |
||||
|
||||
const { Tracing, Profiler } = await this.getClientApis(); |
||||
|
||||
await Promise.all([ |
||||
Tracing.start({ |
||||
bufferUsageReportingInterval: 1000, |
||||
traceConfig: { |
||||
includedCategories: this.tracingCategories, |
||||
}, |
||||
}), |
||||
Profiler.start(), |
||||
]); |
||||
|
||||
Tracing.on('dataCollected', ({ value: events }) => { |
||||
this.state.traceEvents.push(...events); |
||||
}); |
||||
|
||||
let resolveFn: (data: CollectedData) => void; |
||||
this.state.tracingPromise = new Promise<CollectedData>((resolve) => { |
||||
resolveFn = resolve; |
||||
}); |
||||
Tracing.on('tracingComplete', ({ dataLossOccurred }) => { |
||||
const t = new Tracelib(this.state.traceEvents); |
||||
|
||||
const eventCounts = countBy(this.state.traceEvents, (ev) => ev.name); |
||||
|
||||
const fps = t.getFPS(); |
||||
|
||||
resolveFn({ |
||||
eventCounts, |
||||
fps: mean(fps.values), |
||||
tracingDataLoss: dataLossOccurred ? 1 : 0, |
||||
warnings: t.getWarningCounts(), |
||||
}); |
||||
}); |
||||
}; |
||||
|
||||
stop: DataCollector['stop'] = async (req) => { |
||||
if (!this.state.tracingPromise) { |
||||
throw new Error(`collection was never started - there is nothing to stop!`); |
||||
} |
||||
|
||||
const { Tracing, Profiler } = await this.getClientApis(); |
||||
|
||||
// TODO: capture profiler data
|
||||
const [, , traceData] = await Promise.all([Profiler.stop(), Tracing.end(), this.state.tracingPromise]); |
||||
|
||||
await this.resetState(); |
||||
|
||||
return traceData; |
||||
}; |
||||
|
||||
close: DataCollector['close'] = async () => { |
||||
await this.resetState(); |
||||
}; |
||||
} |
||||
@ -0,0 +1,14 @@ |
||||
export type CollectedData = Record<string, unknown>; |
||||
|
||||
export enum DataCollectorName { |
||||
CDP = 'CDP', |
||||
} |
||||
|
||||
type DataCollectorRequest = { id: string }; |
||||
|
||||
export type DataCollector<T extends CollectedData = CollectedData> = { |
||||
start: (input: DataCollectorRequest) => Promise<void>; |
||||
stop: (input: DataCollectorRequest) => Promise<T>; |
||||
getName: () => DataCollectorName; |
||||
close: () => Promise<void>; |
||||
}; |
||||
@ -0,0 +1,138 @@ |
||||
import { CollectedData, DataCollectorName } from './DataCollector'; |
||||
import { fromPairs } from 'lodash'; |
||||
|
||||
type Stats = { |
||||
sum: number; |
||||
min: number; |
||||
max: number; |
||||
count: number; |
||||
avg: number; |
||||
time: number; |
||||
}; |
||||
|
||||
export enum MeasurementName { |
||||
DataRenderDelay = 'DataRenderDelay', |
||||
} |
||||
|
||||
type LivePerformanceAppStats = Record<MeasurementName, Stats[]>; |
||||
|
||||
const isLivePerformanceAppStats = (data: CollectedData[]): data is LivePerformanceAppStats[] => |
||||
data.some((st) => { |
||||
const stat = st?.[MeasurementName.DataRenderDelay]; |
||||
return Array.isArray(stat) && Boolean(stat?.length); |
||||
}); |
||||
|
||||
type FormattedStats = { |
||||
total: { |
||||
count: number[]; |
||||
avg: number[]; |
||||
}; |
||||
lastInterval: { |
||||
avg: number[]; |
||||
min: number[]; |
||||
max: number[]; |
||||
count: number[]; |
||||
}; |
||||
}; |
||||
|
||||
export const formatAppStats = (allStats: CollectedData[]) => { |
||||
if (!isLivePerformanceAppStats(allStats)) { |
||||
return {}; |
||||
} |
||||
|
||||
const names = Object.keys(MeasurementName) as MeasurementName[]; |
||||
|
||||
return fromPairs( |
||||
names.map((name) => { |
||||
const statsForMeasurement = allStats.map((s) => s[name]); |
||||
const res: FormattedStats = { |
||||
total: { |
||||
count: [], |
||||
avg: [], |
||||
}, |
||||
lastInterval: { |
||||
avg: [], |
||||
min: [], |
||||
max: [], |
||||
count: [], |
||||
}, |
||||
}; |
||||
|
||||
statsForMeasurement.forEach((s) => { |
||||
const total = s.reduce( |
||||
(prev, next) => { |
||||
prev.count += next.count; |
||||
prev.avg += next.avg; |
||||
return prev; |
||||
}, |
||||
{ count: 0, avg: 0 } |
||||
); |
||||
res.total.count.push(Math.round(total.count)); |
||||
res.total.avg.push(Math.round(total.avg / s.length)); |
||||
|
||||
const lastInterval = s[s.length - 1]; |
||||
|
||||
res.lastInterval.avg.push(Math.round(lastInterval?.avg)); |
||||
res.lastInterval.min.push(Math.round(lastInterval?.min)); |
||||
res.lastInterval.max.push(Math.round(lastInterval?.max)); |
||||
res.lastInterval.count.push(Math.round(lastInterval?.count)); |
||||
}); |
||||
|
||||
return [name, res]; |
||||
}) |
||||
); |
||||
}; |
||||
|
||||
type CDPData = { |
||||
eventCounts: Record<string, unknown>; |
||||
fps: number; |
||||
tracingDataLoss: number; |
||||
warnings: Record<string, unknown>; |
||||
}; |
||||
|
||||
const isCDPData = (data: any[]): data is CDPData[] => data.every((d) => typeof d.eventCounts === 'object'); |
||||
|
||||
type FormattedCDPData = { |
||||
minorGC: number[]; |
||||
majorGC: number[]; |
||||
droppedFrames: number[]; |
||||
fps: number[]; |
||||
tracingDataLossOccurred: boolean; |
||||
longTaskWarnings: number[]; |
||||
}; |
||||
|
||||
const emptyFormattedCDPData = (): FormattedCDPData => ({ |
||||
minorGC: [], |
||||
majorGC: [], |
||||
droppedFrames: [], |
||||
fps: [], |
||||
tracingDataLossOccurred: false, |
||||
longTaskWarnings: [], |
||||
}); |
||||
|
||||
const formatCDPData = (data: any): FormattedCDPData => { |
||||
if (!isCDPData(data)) { |
||||
return emptyFormattedCDPData(); |
||||
} |
||||
|
||||
return data.reduce((acc, next) => { |
||||
acc.majorGC.push((next.eventCounts.MajorGC as number) ?? 0); |
||||
acc.minorGC.push((next.eventCounts.MinorGC as number) ?? 0); |
||||
acc.fps.push(Math.round(next.fps) ?? 0); |
||||
acc.tracingDataLossOccurred = acc.tracingDataLossOccurred || Boolean(next.tracingDataLoss); |
||||
acc.droppedFrames.push((next.eventCounts.DroppedFrame as number) ?? 0); |
||||
acc.longTaskWarnings.push((next.warnings.LongTask as number) ?? 0); |
||||
return acc; |
||||
}, emptyFormattedCDPData()); |
||||
}; |
||||
|
||||
export const formatResults = ( |
||||
results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> |
||||
): CollectedData => { |
||||
return { |
||||
...formatAppStats(results.map(({ appStats }) => appStats)), |
||||
...formatCDPData(results.map(({ collectorsData }) => collectorsData[DataCollectorName.CDP])), |
||||
|
||||
__raw: results, |
||||
}; |
||||
}; |
||||
@ -0,0 +1,87 @@ |
||||
import { CollectedData, DataCollector } from './DataCollector'; |
||||
import { CDPDataCollector } from './CDPDataCollector'; |
||||
import { fromPairs } from 'lodash'; |
||||
import fs from 'fs'; |
||||
import { formatResults } from './formatting'; |
||||
const remoteDebuggingPortOptionPrefix = '--remote-debugging-port='; |
||||
|
||||
const getOrAddRemoteDebuggingPort = (args: string[]) => { |
||||
const existing = args.find((arg) => arg.startsWith(remoteDebuggingPortOptionPrefix)); |
||||
|
||||
if (existing) { |
||||
return Number(existing.substring(remoteDebuggingPortOptionPrefix.length)); |
||||
} |
||||
|
||||
const port = 40000 + Math.round(Math.random() * 25000); |
||||
args.push(`${remoteDebuggingPortOptionPrefix}${port}`); |
||||
return port; |
||||
}; |
||||
|
||||
let collectors: DataCollector[] = []; |
||||
let results: Array<{ appStats: CollectedData; collectorsData: CollectedData }> = []; |
||||
|
||||
const startBenchmarking = async ({ testName }: { testName: string }) => { |
||||
await Promise.all(collectors.map((coll) => coll.start({ id: testName }))); |
||||
|
||||
return true; |
||||
}; |
||||
|
||||
const stopBenchmarking = async ({ testName, appStats }: { testName: string; appStats: CollectedData }) => { |
||||
const data = await Promise.all(collectors.map(async (coll) => [coll.getName(), await coll.stop({ id: testName })])); |
||||
|
||||
results.push({ |
||||
collectorsData: fromPairs(data), |
||||
appStats: appStats, |
||||
}); |
||||
|
||||
return true; |
||||
}; |
||||
const afterRun = async () => { |
||||
await Promise.all(collectors.map((coll) => coll.close())); |
||||
collectors = []; |
||||
results = []; |
||||
}; |
||||
|
||||
const afterSpec = (resultsFolder: string) => async (spec: { name: string }) => { |
||||
fs.writeFileSync(`${resultsFolder}/${spec.name}-${Date.now()}.json`, JSON.stringify(formatResults(results), null, 2)); |
||||
|
||||
results = []; |
||||
}; |
||||
|
||||
export const initialize: Cypress.PluginConfig = (on, config) => { |
||||
const resultsFolder = config.env['BENCHMARK_PLUGIN_RESULTS_FOLDER']; |
||||
|
||||
if (!fs.existsSync(resultsFolder)) { |
||||
fs.mkdirSync(resultsFolder, { recursive: true }); |
||||
console.log(`Created folder for benchmark results ${resultsFolder}`); |
||||
} |
||||
|
||||
on('before:browser:launch', async (browser, options) => { |
||||
if (browser.family !== 'chromium' || browser.name === 'electron') { |
||||
throw new Error('benchmarking plugin requires chrome'); |
||||
} |
||||
|
||||
const { args } = options; |
||||
|
||||
const port = getOrAddRemoteDebuggingPort(args); |
||||
collectors.push(new CDPDataCollector({ port })); |
||||
|
||||
args.push('--start-fullscreen'); |
||||
|
||||
console.log( |
||||
`initialized benchmarking plugin with ${collectors.length} collectors: ${collectors |
||||
.map((col) => col.getName()) |
||||
.join(', ')}` |
||||
); |
||||
|
||||
return options; |
||||
}); |
||||
|
||||
on('task', { |
||||
startBenchmarking, |
||||
stopBenchmarking, |
||||
}); |
||||
|
||||
on('after:run', afterRun); |
||||
on('after:spec', afterSpec(resultsFolder)); |
||||
}; |
||||
@ -0,0 +1,15 @@ |
||||
type TraceEvent = { |
||||
name: string; |
||||
}; |
||||
|
||||
declare class Tracelib { |
||||
constructor(private events: TraceEvent[]) {} |
||||
|
||||
getFPS: () => { times: number[]; values: number[] }; |
||||
getWarningCounts: () => Record<string, number>; |
||||
} |
||||
declare module 'tracelib' { |
||||
export = Tracelib; |
||||
|
||||
export { TraceEvent }; |
||||
} |
||||
@ -0,0 +1,81 @@ |
||||
import { e2e } from '../'; |
||||
|
||||
export interface BenchmarkArguments { |
||||
name: string; |
||||
dashboard: { |
||||
folder: string; |
||||
delayAfterOpening: number; |
||||
skipPanelValidation: boolean; |
||||
}; |
||||
repeat: number; |
||||
duration: number; |
||||
appStats?: { |
||||
startCollecting?: (window: Window) => void; |
||||
collect: (window: Window) => Record<string, unknown>; |
||||
}; |
||||
skipScenario?: boolean; |
||||
} |
||||
|
||||
export const benchmark = ({ |
||||
name, |
||||
skipScenario = false, |
||||
repeat, |
||||
duration, |
||||
appStats, |
||||
dashboard, |
||||
}: BenchmarkArguments) => { |
||||
if (skipScenario) { |
||||
describe(name, () => { |
||||
it.skip(name, () => {}); |
||||
}); |
||||
} |
||||
|
||||
describe(name, () => { |
||||
before(() => { |
||||
e2e.flows.login(e2e.env('USERNAME'), e2e.env('PASSWORD')); |
||||
}); |
||||
|
||||
beforeEach(() => { |
||||
e2e.flows.importDashboards(dashboard.folder, 1000, dashboard.skipPanelValidation); |
||||
Cypress.Cookies.preserveOnce('grafana_session'); |
||||
}); |
||||
|
||||
afterEach(() => e2e.flows.revertAllChanges()); |
||||
after(() => { |
||||
e2e().clearCookies(); |
||||
}); |
||||
|
||||
Array(repeat) |
||||
.fill(0) |
||||
.map((_, i) => { |
||||
const testName = `${name}-${i}`; |
||||
return it(testName, () => { |
||||
e2e.flows.openDashboard(); |
||||
|
||||
e2e().wait(dashboard.delayAfterOpening); |
||||
|
||||
if (appStats) { |
||||
const startCollecting = appStats.startCollecting; |
||||
if (startCollecting) { |
||||
e2e() |
||||
.window() |
||||
.then((win) => startCollecting(win)); |
||||
} |
||||
|
||||
e2e().startBenchmarking(testName); |
||||
e2e().wait(duration); |
||||
|
||||
e2e() |
||||
.window() |
||||
.then((win) => { |
||||
e2e().stopBenchmarking(testName, appStats.collect(win)); |
||||
}); |
||||
} else { |
||||
e2e().startBenchmarking(testName); |
||||
e2e().wait(duration); |
||||
e2e().stopBenchmarking(testName, {}); |
||||
} |
||||
}); |
||||
}); |
||||
}); |
||||
}; |
||||
Loading…
Reference in new issue