FrontendMetrics: Adds new backend api that frontend can use to push frontend measurements and counters to prometheus (#32593)

* FrontendMetrics: Adds new backend api that frontend can use to push frontend measurements and counters to prometheus

* FrontendMetrics: Adds new backend api that frontend can use to push frontend measurements and counters to prometheus

* Fix naming

* change to histogram

* Fixed go lint

devenv/docker/blocks/prometheus2/docker-compose.yaml

@@ -2,6 +2,8 @@
     build: docker/blocks/prometheus2
     ports:
       - "9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   node_exporter:
     image: prom/node-exporter

devenv/docker/blocks/prometheus2/prometheus.yml

@@ -32,7 +32,7 @@ scrape_configs:
 
   - job_name: 'grafana'
     static_configs:
-      - targets: ['127.0.0.1:3000']
+      - targets: ['host.docker.internal:3000']
 
   - job_name: 'prometheus-random-data'
     static_configs:

pkg/api/api.go

@@ -10,6 +10,7 @@ import (
 	"github.com/grafana/grafana/pkg/api/frontendlogging"
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/models"
 )
@@ -397,6 +398,8 @@ func (hs *HTTPServer) registerRoutes() {
 			annotationsRoute.Post("/graphite", reqEditorRole, bind(dtos.PostGraphiteAnnotationsCmd{}), routing.Wrap(PostGraphiteAnnotation))
 		})
 
+		apiRoute.Post("/frontend-metrics", bind(metrics.PostFrontendMetricsCommand{}), routing.Wrap(hs.PostFrontendMetrics))
+
 		if hs.Live.IsEnabled() {
 			apiRoute.Post("/live/publish", bind(dtos.LivePublishCmd{}), routing.Wrap(hs.Live.HandleHTTPPublish))
 		}

pkg/api/frontend_metrics.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"strings"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+func (hs *HTTPServer) PostFrontendMetrics(c *models.ReqContext, cmd metrics.PostFrontendMetricsCommand) response.Response {
+	for _, event := range cmd.Events {
+		name := strings.Replace(event.Name, "-", "_", -1)
+		if recorder, ok := metrics.FrontendMetrics[name]; ok {
+			recorder(event)
+		} else {
+			c.Logger.Debug("Received unknown frontend metric", "metric", name)
+		}
+	}
+	return response.Empty(200)
+}

pkg/infra/metrics/frontendmetrics.go

@@ -0,0 +1,43 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// PostPostFrontendMetricsCommand sent by frontend to record frontend metrics
+type PostFrontendMetricsCommand struct {
+	Events []FrontendMetricEvent `json:"events"`
+}
+
+// FrontendMetricEvent a single metric measurement event
+type FrontendMetricEvent struct {
+	Name  string  `json:"name"`
+	Value float64 `json:"value"`
+}
+
+// FrontendMetricsRecorder handles the recording of the event, ie passes it to a prometheus metric
+type FrontendMetricsRecorder func(event FrontendMetricEvent)
+
+// FrontendMetrics contains all the valid frontend metrics and a handler function for recording events
+var FrontendMetrics map[string]FrontendMetricsRecorder = map[string]FrontendMetricsRecorder{}
+
+func registerFrontendHistogram(name string, help string) {
+	defBuckets := []float64{.1, .25, .5, 1, 1.5, 2, 5, 10, 20, 40}
+
+	histogram := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:      name,
+		Help:      help,
+		Buckets:   defBuckets,
+		Namespace: ExporterName,
+	})
+
+	FrontendMetrics[name] = func(event FrontendMetricEvent) {
+		histogram.Observe(event.Value)
+	}
+
+	prometheus.MustRegister(histogram)
+}
+
+func initFrontendMetrics() {
+	registerFrontendHistogram("frontend_boot_load_time_seconds", "Frontend boot time measurement")
+	registerFrontendHistogram("frontend_boot_first_paint_time_seconds", "Frontend boot first paint")
+	registerFrontendHistogram("frontend_boot_js_done_time_seconds", "Frontend boot initial js load")
+}

pkg/infra/metrics/service.go

@@ -22,6 +22,7 @@ func (lw *logWrapper) Println(v ...interface{}) {
 func init() {
 	registry.RegisterService(&InternalMetricsService{})
 	initMetricVars()
+	initFrontendMetrics()
 }
 
 type InternalMetricsService struct {

public/app/app.ts

@@ -124,18 +124,22 @@ function initEchoSrv() {
   setEchoSrv(new Echo({ debug: process.env.NODE_ENV === 'development' }));
 
   window.addEventListener('load', (e) => {
-    // Collecting paint metrics first
+    const loadMetricName = 'frontend_boot_load_time_seconds';
+
     if (performance && performance.getEntriesByType) {
-      performance.mark('load');
+      performance.mark(loadMetricName);
 
       const paintMetrics = performance.getEntriesByType('paint');
 
       for (const metric of paintMetrics) {
-        reportPerformance(metric.name, Math.round(metric.startTime + metric.duration));
+        reportPerformance(
+          `frontend_boot_${metric.name}_time_seconds`,
+          Math.round(metric.startTime + metric.duration) / 1000
+        );
       }
 
-      const loadMetric = performance.getEntriesByName('load')[0];
-      reportPerformance(loadMetric.name, Math.round(loadMetric.startTime + loadMetric.duration));
+      const loadMetric = performance.getEntriesByName(loadMetricName)[0];
+      reportPerformance(loadMetric.name, Math.round(loadMetric.startTime + loadMetric.duration) / 1000);
     }
   });
 
@@ -150,10 +154,6 @@ function initEchoSrv() {
       })
     );
   }
-
-  window.addEventListener('DOMContentLoaded', () => {
-    reportPerformance('dcl', Math.round(performance.now()));
-  });
 }
 
 function addClassIfNoOverlayScrollbar() {

public/app/core/services/echo/EchoSrv.ts

@@ -6,8 +6,8 @@ export const reportPerformance = (metric: string, value: number) => {
   getEchoSrv().addEvent<PerformanceEvent>({
     type: EchoEventType.Performance,
     payload: {
-      metricName: metric,
-      duration: value,
+      name: metric,
+      value: value,
     },
   });
 };

public/app/core/services/echo/backends/PerformanceBackend.ts

@@ -1,8 +1,9 @@
 import { EchoBackend, EchoEvent, EchoEventType } from '@grafana/runtime';
+import { backendSrv } from '../../backend_srv';
 
 export interface PerformanceEventPayload {
-  metricName: string;
-  duration: number;
+  name: string;
+  value: number;
 }
 
 export interface PerformanceEvent extends EchoEvent<EchoEventType.Performance, PerformanceEventPayload> {}
@@ -16,13 +17,13 @@ export interface PerformanceBackendOptions {
  * Reports performance metrics to given url (TODO)
  */
 export class PerformanceBackend implements EchoBackend<PerformanceEvent, PerformanceBackendOptions> {
-  private buffer: PerformanceEvent[] = [];
+  private buffer: PerformanceEventPayload[] = [];
   supportedEvents = [EchoEventType.Performance];
 
   constructor(public options: PerformanceBackendOptions) {}
 
   addEvent = (e: EchoEvent) => {
-    this.buffer.push(e);
+    this.buffer.push(e.payload);
   };
 
   flush = () => {
@@ -30,20 +31,17 @@ export class PerformanceBackend implements EchoBackend<PerformanceEvent, Perform
       return;
     }
 
-    const result = {
-      metrics: this.buffer,
-    };
-
     // Currently we don't have an API for sending the metrics hence logging to console in dev environment
     if (process.env.NODE_ENV === 'development') {
-      console.log('PerformanceBackend flushing:', result);
+      console.log('PerformanceBackend flushing:', this.buffer);
     }
 
-    this.buffer = [];
+    console.log('performance', this.buffer);
 
-    // TODO: Enable backend request when we have metrics API
-    // if (this.options.url) {
-    // backendSrv.post(this.options.url, result);
-    // }
+    backendSrv.post('/api/frontend-metrics', {
+      events: this.buffer,
+    });
+
+    this.buffer = [];
   };
 }

public/views/index-template.html

@@ -23,8 +23,9 @@
     <link rel="stylesheet" href="[[.ContentDeliveryURL]]public/build/grafana.[[ .Theme ]].<%= webpack.hash %>.css" />
 
     <script nonce="[[.Nonce]]">
-      performance.mark('css done blocking');
+      performance.mark('frontend_boot_css_time_seconds');
     </script>
+
     <meta name="apple-mobile-web-app-capable" content="yes" />
     <meta name="apple-mobile-web-app-status-bar-style" content="black" />
     <meta name="msapplication-TileColor" content="#2b5797" />
@@ -314,8 +315,9 @@
       type="text/javascript"
     ></script>
     <% } %> <% } %>
+
     <script nonce="[[.Nonce]]">
-      performance.mark('js done blocking');
+      performance.mark('frontend_boot_js_done_time_seconds');
     </script>
   </body>
 </html>