web: add slo group to request meta (#74765)

Signed-off-by: bergquist <carl.bergquist@gmail.com>

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -138,6 +138,7 @@ Experimental features might be changed or removed without prior notice.
 | `alertingInsights`                          | Show the new alerting insights landing page                                                                  |
 | `externalCorePlugins`                       | Allow core plugins to be loaded as external                                                                  |
 | `pluginsAPIMetrics`                         | Sends metrics of public grafana packages usage by plugins                                                    |
+| `httpSLOLevels`                             | Adds SLO level to http request metrics                                                                       |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -129,4 +129,5 @@ export interface FeatureToggles {
   alertingInsights?: boolean;
   externalCorePlugins?: boolean;
   pluginsAPIMetrics?: boolean;
+  httpSLOLevels?: boolean;
 }

pkg/api/api.go

@@ -436,19 +436,19 @@ func (hs *HTTPServer) registerRoutes() {
 		}
 
 		apiRoute.Get("/frontend/settings/", hs.GetFrontendSettings)
-		apiRoute.Any("/datasources/proxy/:id/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest)
-		apiRoute.Any("/datasources/proxy/uid/:uid/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID)
-		apiRoute.Any("/datasources/proxy/:id", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest)
-		apiRoute.Any("/datasources/proxy/uid/:uid", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID)
+		apiRoute.Any("/datasources/proxy/:id/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest)
+		apiRoute.Any("/datasources/proxy/uid/:uid/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID)
+		apiRoute.Any("/datasources/proxy/:id", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest)
+		apiRoute.Any("/datasources/proxy/uid/:uid", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID)
 		// Deprecated: use /datasources/uid/:uid/resources API instead.
-		apiRoute.Any("/datasources/:id/resources", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource)
-		apiRoute.Any("/datasources/uid/:uid/resources", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID)
+		apiRoute.Any("/datasources/:id/resources", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource)
+		apiRoute.Any("/datasources/uid/:uid/resources", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID)
 		// Deprecated: use /datasources/uid/:uid/resources/* API instead.
-		apiRoute.Any("/datasources/:id/resources/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource)
-		apiRoute.Any("/datasources/uid/:uid/resources/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID)
+		apiRoute.Any("/datasources/:id/resources/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource)
+		apiRoute.Any("/datasources/uid/:uid/resources/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID)
 		// Deprecated: use /datasources/uid/:uid/health API instead.
-		apiRoute.Any("/datasources/:id/health", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealth))
-		apiRoute.Any("/datasources/uid/:uid/health", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealthWithUID))
+		apiRoute.Any("/datasources/:id/health", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealth))
+		apiRoute.Any("/datasources/uid/:uid/health", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealthWithUID))
 
 		// Folders
 		apiRoute.Group("/folders", func(folderRoute routing.RouteRegister) {
@@ -532,7 +532,7 @@ func (hs *HTTPServer) registerRoutes() {
 
 		// metrics
 		// DataSource w/ expressions
-		apiRoute.Post("/ds/query", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.QueryMetricsV2))
+		apiRoute.Post("/ds/query", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.QueryMetricsV2))
 
 		apiRoute.Group("/alerts", func(alertsRoute routing.RouteRegister) {
 			alertsRoute.Post("/test", routing.Wrap(hs.AlertTest))
@@ -597,7 +597,7 @@ func (hs *HTTPServer) registerRoutes() {
 
 			// Some channels may have info
 			liveRoute.Get("/info/*", routing.Wrap(hs.Live.HandleInfoHTTP))
-		})
+		}, requestmeta.SetSLOGroup(requestmeta.SLOGroupNone))
 
 		// short urls
 		apiRoute.Post("/short-urls", routing.Wrap(hs.createShortURL))

pkg/middleware/request_metrics.go

@@ -45,6 +45,10 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR
 		histogramLabels = append(histogramLabels, "team")
 	}
 
+	if features.IsEnabled(featuremgmt.FlagHttpSLOLevels) {
+		histogramLabels = append(histogramLabels, "slo_group")
+	}
+
 	httpRequestDurationHistogram := prometheus.NewHistogramVec(
 		prometheus.HistogramOpts{
 			Namespace: "grafana",
@@ -95,6 +99,10 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR
 				labelValues = append(labelValues, rmd.Team)
 			}
 
+			if features.IsEnabled(featuremgmt.FlagHttpSLOLevels) {
+				labelValues = append(labelValues, string(rmd.SLOGroup))
+			}
+
 			// avoiding the sanitize functions for in the new instrumentation
 			// since they dont make much sense. We should remove them later.
 			histogram := httpRequestDurationHistogram.

pkg/middleware/requestmeta/request_metadata.go

@@ -20,13 +20,38 @@ const (
 	StatusSourceDownstream StatusSource = "downstream"
 )
 
-type rMDContextKey struct{}
-
 type RequestMetaData struct {
 	Team         string
 	StatusSource StatusSource
+	SLOGroup     SLOGroup
 }
 
+type SLOGroup string
+
+const (
+	// SLOGroupHighFast is the default slo group for handlers in Grafana
+	// Most handlers should respond quickly
+	SLOGroupHighFast SLOGroup = "high-fast"
+
+	// SLOGroupHighMedium is used for handlers that might take some
+	// time to respond.
+	SLOGroupHighMedium SLOGroup = "high-medium"
+
+	// SLOGroupHighSlow is used by handlers that proxies requests downstream.
+	// We expect an high successrate but without latency garantess
+	SLOGroupHighSlow SLOGroup = "high-slow"
+
+	// SLOGroupLow should only be used for experimental features
+	// that might not be stable enough for our normal SLO
+	SLOGroupLow SLOGroup = "low"
+
+	// SLOGroupNone means that errors are expect for this handler and
+	// it should not be included in the SLO.
+	SLOGroupNone SLOGroup = "none"
+)
+
+type rMDContextKey struct{}
+
 var requestMetaDataContextKey = rMDContextKey{}
 
 // SetupRequestMetadata injects defaul request metadata values
@@ -96,5 +121,14 @@ func defaultRequestMetadata() RequestMetaData {
 	return RequestMetaData{
 		Team:         TeamCore,
 		StatusSource: StatusSourceServer,
+		SLOGroup:     SLOGroupHighFast,
+	}
+}
+
+// SetSLOGroup creates an middleware that sets the SLOGroup for the request
+func SetSLOGroup(lvl SLOGroup) web.Handler {
+	return func(w http.ResponseWriter, r *http.Request) {
+		v := GetRequestMetaData(r.Context())
+		v.SLOGroup = lvl
 	}
 }

pkg/services/featuremgmt/registry.go

@@ -772,5 +772,13 @@ var (
 			Stage:        FeatureStageExperimental,
 			Owner:        grafanaPluginsPlatformSquad,
 		},
+		{
+			Name:            "httpSLOLevels",
+			Description:     "Adds SLO level to http request metrics",
+			Stage:           FeatureStageExperimental,
+			FrontendOnly:    false,
+			Owner:           hostedGrafanaTeam,
+			RequiresRestart: true,
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -110,3 +110,4 @@ wargamesTesting,experimental,@grafana/hosted-grafana-team,false,false,false,fals
 alertingInsights,experimental,@grafana/alerting-squad,false,false,false,true
 externalCorePlugins,experimental,@grafana/plugins-platform-backend,false,false,false,false
 pluginsAPIMetrics,experimental,@grafana/plugins-platform-backend,false,false,false,true
+httpSLOLevels,experimental,@grafana/hosted-grafana-team,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -450,4 +450,8 @@ const (
 	// FlagPluginsAPIMetrics
 	// Sends metrics of public grafana packages usage by plugins
 	FlagPluginsAPIMetrics = "pluginsAPIMetrics"
+
+	// FlagHttpSLOLevels
+	// Adds SLO level to http request metrics
+	FlagHttpSLOLevels = "httpSLOLevels"
 )