Dashboards: Monitor dashboard loading performance (#99629)

* WIP benchmark dashboard rendering

* Script

* Benchmark with variable and a panel

* Add one more benchmark

* Explicitely enable profiling

* Playwright tests

* update scenes

* Report measurement to faro when config set

* Let user enable metrics reporting in UI

* Fix logging

* Change how performance metrics is enabled per dashboard, now in config file only

* add benchmark run option

* Fix benchmark runs

* fix description for performance config

* remove console.log

* update codeowners

* add back crashDetection init that was lost in merge

* fix yarn.lock

* restore custom.ini

* fix import

* Make sure we have the echoSrv

* fix config type

* Try to limit changes to e2e runs

* remove benchmark

* Fix lint issue

* fix codeowners

---------

Co-authored-by: Dominik Prokop <dominik.prokop@grafana.com>
Co-authored-by: Sergej-Vlasov <sergej.s.vlasov@gmail.com>

.gitignore

@@ -176,7 +176,6 @@ compilation-stats.json
 /blob-report/
 /playwright/.cache/
 /playwright/.auth/
-
 # grafana server
 /scripts/grafana-server/server.log
 

conf/defaults.ini

@@ -452,6 +452,9 @@ min_refresh_interval = 5s
 # Path to the default home dashboard. If this value is empty, then Grafana uses StaticRootPath + "dashboards/home.json"
 default_home_dashboard_path =
 
+# Dashboards UIDs to report performance metrics for. * can be used to report metrics for all dashboards
+dashboard_performance_metrics =
+
 ################################### Data sources #########################
 [datasources]
 # Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API.

package.json

@@ -18,6 +18,8 @@
     "e2e:enterprise": "./e2e/start-and-run-suite enterprise",
     "e2e:enterprise:dev": "./e2e/start-and-run-suite enterprise dev",
     "e2e:enterprise:debug": "./e2e/start-and-run-suite enterprise debug",
+    "build-benchmark": "NODE_ENV=dev nx exec -- webpack --config scripts/webpack/webpack.dev.js --env benchmark=1",
+    "e2e:playwright:benchmark": "yarn build-benchmark && ./e2e/plugin-e2e/start-and-benchmark",
     "e2e:playwright": "yarn playwright test",
     "e2e:playwright:server": "yarn e2e:plugin:build && ./e2e/plugin-e2e/start-and-run-suite",
     "e2e:storybook": "PORT=9001 ./e2e/run-suite storybook true",

packages/grafana-data/src/types/config.ts

@@ -228,6 +228,7 @@ export interface GrafanaConfig {
   rudderstackConfigUrl: string | undefined;
   rudderstackIntegrationsUrl: string | undefined;
   analyticsConsoleReporting: boolean;
+  dashboardPerformanceMetrics: string[];
   sqlConnectionLimits: SqlConnectionLimits;
   sharedWithMeFolderUID?: string;
   rootFolderUID?: string;

packages/grafana-runtime/src/config.ts

@@ -185,6 +185,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
   rudderstackConfigUrl: undefined;
   rudderstackIntegrationsUrl: undefined;
   analyticsConsoleReporting = false;
+  dashboardPerformanceMetrics: string[] = [];
   sqlConnectionLimits = {
     maxOpenConns: 100,
     maxIdleConns: 100,

packages/grafana-runtime/src/index.ts

@@ -9,7 +9,7 @@ export * from './analytics/types';
 export { loadPluginCss, type PluginCssOptions, setPluginImportUtils, getPluginImportUtils } from './utils/plugin';
 export { reportMetaAnalytics, reportInteraction, reportPageview, reportExperimentView } from './analytics/utils';
 export { featureEnabled } from './utils/licensing';
-export { logInfo, logDebug, logWarning, logError, createMonitoringLogger } from './utils/logging';
+export { logInfo, logDebug, logWarning, logError, createMonitoringLogger, logMeasurement } from './utils/logging';
 export {
   DataSourceWithBackend,
   HealthCheckError,

packages/grafana-runtime/src/utils/logging.ts

@@ -66,11 +66,13 @@ export function logError(err: Error, contexts?: LogContext) {
 export type MeasurementValues = Record<string, number>;
 export function logMeasurement(type: string, values: MeasurementValues, context?: LogContext) {
   if (config.grafanaJavascriptAgent.enabled) {
-    faro.api.pushMeasurement({
-      type,
-      values,
-      context,
-    });
+    faro.api.pushMeasurement(
+      {
+        type,
+        values,
+      },
+      { context: context }
+    );
   }
 }
 

pkg/api/dtos/frontend_settings.go

@@ -190,6 +190,8 @@ type FrontendSettingsDTO struct {
 
 	AnalyticsConsoleReporting bool `json:"analyticsConsoleReporting"`
 
+	DashboardPerformanceMetrics []string `json:"dashboardPerformanceMetrics"`
+
 	FeedbackLinksEnabled                bool     `json:"feedbackLinksEnabled"`
 	ApplicationInsightsConnectionString string   `json:"applicationInsightsConnectionString"`
 	ApplicationInsightsEndpointUrl      string   `json:"applicationInsightsEndpointUrl"`

pkg/api/frontendsettings.go

@@ -215,6 +215,7 @@ func (hs *HTTPServer) getFrontendSettings(c *contextmodel.ReqContext) (*dtos.Fro
 		RudderstackConfigUrl:                hs.Cfg.RudderstackConfigURL,
 		RudderstackIntegrationsUrl:          hs.Cfg.RudderstackIntegrationsURL,
 		AnalyticsConsoleReporting:           hs.Cfg.FrontendAnalyticsConsoleReporting,
+		DashboardPerformanceMetrics:         hs.Cfg.DashboardPerformanceMetrics,
 		FeedbackLinksEnabled:                hs.Cfg.FeedbackLinksEnabled,
 		ApplicationInsightsConnectionString: hs.Cfg.ApplicationInsightsConnectionString,
 		ApplicationInsightsEndpointUrl:      hs.Cfg.ApplicationInsightsEndpointUrl,

pkg/setting/setting.go

@@ -218,9 +218,10 @@ type Cfg struct {
 	MetricsGrafanaEnvironmentInfo    map[string]string
 
 	// Dashboards
-	DashboardVersionsToKeep  int
-	MinRefreshInterval       string
-	DefaultHomeDashboardPath string
+	DashboardVersionsToKeep     int
+	MinRefreshInterval          string
+	DefaultHomeDashboardPath    string
+	DashboardPerformanceMetrics []string
 
 	// Auth
 	LoginCookieName               string
@@ -1133,6 +1134,7 @@ func (cfg *Cfg) parseINIFile(iniFile *ini.File) error {
 	cfg.DashboardVersionsToKeep = dashboards.Key("versions_to_keep").MustInt(20)
 	cfg.MinRefreshInterval = valueAsString(dashboards, "min_refresh_interval", "5s")
 	cfg.DefaultHomeDashboardPath = dashboards.Key("default_home_dashboard_path").MustString("")
+	cfg.DashboardPerformanceMetrics = util.SplitString(dashboards.Key("dashboard_performance_metrics").MustString(""))
 
 	if err := readUserSettings(iniFile, cfg); err != nil {
 		return err

public/app/features/dashboard-scene/pages/DashboardScenePageStateManager.ts

@@ -2,6 +2,7 @@ import { isEqual } from 'lodash';
 
 import { locationUtil, UrlQueryMap } from '@grafana/data';
 import { config, getBackendSrv, isFetchError, locationService } from '@grafana/runtime';
+import { sceneGraph } from '@grafana/scenes';
 import { DashboardV2Spec } from '@grafana/schema/dist/esm/schema/dashboard/v2alpha0';
 import { StateManagerBase } from 'app/core/services/StateManagerBase';
 import { getMessageFromError, getMessageIdFromError, getStatusFromError } from 'app/core/utils/errors';
@@ -135,7 +136,10 @@ abstract class DashboardScenePageStateManagerBase<T>
 
       this.setState({ dashboard: dashboard, isLoading: false, options });
       const measure = stopMeasure(LOAD_SCENE_MEASUREMENT);
+      const queryController = sceneGraph.getQueryController(dashboard);
+
       trackDashboardSceneLoaded(dashboard, measure?.duration);
+      queryController?.startProfile(dashboard);
 
       if (options.route !== DashboardRoutes.New) {
         emitDashboardViewEvent({

public/app/features/dashboard-scene/serialization/transformSaveModelToScene.ts

@@ -1,7 +1,7 @@
 import { uniqueId } from 'lodash';
 
 import { DataFrameDTO, DataFrameJSON } from '@grafana/data';
-import { config } from '@grafana/runtime';
+import { config, logMeasurement, reportInteraction } from '@grafana/runtime';
 import {
   VizPanel,
   SceneTimePicker,
@@ -19,6 +19,7 @@ import {
   SceneDataLayerProvider,
   SceneDataLayerControls,
   UserActionEvent,
+  SceneInteractionProfileEvent,
   SceneObjectState,
 } from '@grafana/scenes';
 import { contextSrv } from 'app/core/core';
@@ -229,7 +230,11 @@ export function createDashboardSceneFromDashboardModel(oldModel: DashboardModel,
     new behaviors.CursorSync({
       sync: oldModel.graphTooltip,
     }),
-    new behaviors.SceneQueryController(),
+    new behaviors.SceneQueryController({
+      enableProfiling:
+        config.dashboardPerformanceMetrics.findIndex((uid) => uid === '*' || uid === oldModel.uid) !== -1,
+      onProfileComplete: getDashboardInteractionCallback(oldModel.uid, oldModel.title),
+    }),
     registerDashboardMacro,
     registerPanelInteractionsReporter,
     new behaviors.LiveNowTimer({ enabled: oldModel.liveNow }),
@@ -434,3 +439,40 @@ function trackIfEmpty(grid: SceneGridLayout) {
     sub.unsubscribe();
   };
 }
+
+function getDashboardInteractionCallback(uid: string, title: string) {
+  return (e: SceneInteractionProfileEvent) => {
+    let interactionType = '';
+
+    if (e.origin === 'SceneTimeRange') {
+      interactionType = 'time-range-change';
+    } else if (e.origin === 'SceneRefreshPicker') {
+      interactionType = 'refresh';
+    } else if (e.origin === 'DashboardScene') {
+      interactionType = 'view';
+    } else if (e.origin.indexOf('Variable') > -1) {
+      interactionType = 'variable-change';
+    }
+    reportInteraction('dashboard-render', {
+      interactionType,
+      duration: e.duration,
+      networkDuration: e.networkDuration,
+      totalJSHeapSize: e.totalJSHeapSize,
+      usedJSHeapSize: e.usedJSHeapSize,
+      jsHeapSizeLimit: e.jsHeapSizeLimit,
+    });
+
+    logMeasurement(
+      `dashboard.${interactionType}`,
+      {
+        duration: e.duration,
+        networkDuration: e.networkDuration,
+        totalJSHeapSize: e.totalJSHeapSize,
+        usedJSHeapSize: e.usedJSHeapSize,
+        jsHeapSizeLimit: e.jsHeapSizeLimit,
+        timeSinceBoot: performance.measure('time_since_boot', 'frontend_boot_js_done_time_seconds').duration,
+      },
+      { dashboard: uid, title: title }
+    );
+  };
+}