diff --git a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md index f98d9a05e76..193ef5f77f0 100644 --- a/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md +++ b/docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md @@ -138,6 +138,7 @@ Experimental features might be changed or removed without prior notice. | `alertingInsights` | Show the new alerting insights landing page | | `externalCorePlugins` | Allow core plugins to be loaded as external | | `pluginsAPIMetrics` | Sends metrics of public grafana packages usage by plugins | +| `httpSLOLevels` | Adds SLO level to http request metrics | ## Development feature toggles diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index d9aaca3f322..9142cca7933 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -129,4 +129,5 @@ export interface FeatureToggles { alertingInsights?: boolean; externalCorePlugins?: boolean; pluginsAPIMetrics?: boolean; + httpSLOLevels?: boolean; } diff --git a/pkg/api/api.go b/pkg/api/api.go index a3b17e46374..62044d88463 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -436,19 +436,19 @@ func (hs *HTTPServer) registerRoutes() { } apiRoute.Get("/frontend/settings/", hs.GetFrontendSettings) - apiRoute.Any("/datasources/proxy/:id/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest) - apiRoute.Any("/datasources/proxy/uid/:uid/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID) - apiRoute.Any("/datasources/proxy/:id", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest) - apiRoute.Any("/datasources/proxy/uid/:uid", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID) + apiRoute.Any("/datasources/proxy/:id/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest) + apiRoute.Any("/datasources/proxy/uid/:uid/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID) + apiRoute.Any("/datasources/proxy/:id", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequest) + apiRoute.Any("/datasources/proxy/uid/:uid", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.ProxyDataSourceRequestWithUID) // Deprecated: use /datasources/uid/:uid/resources API instead. - apiRoute.Any("/datasources/:id/resources", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource) - apiRoute.Any("/datasources/uid/:uid/resources", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID) + apiRoute.Any("/datasources/:id/resources", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource) + apiRoute.Any("/datasources/uid/:uid/resources", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID) // Deprecated: use /datasources/uid/:uid/resources/* API instead. - apiRoute.Any("/datasources/:id/resources/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource) - apiRoute.Any("/datasources/uid/:uid/resources/*", authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID) + apiRoute.Any("/datasources/:id/resources/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResource) + apiRoute.Any("/datasources/uid/:uid/resources/*", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), hs.CallDatasourceResourceWithUID) // Deprecated: use /datasources/uid/:uid/health API instead. - apiRoute.Any("/datasources/:id/health", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealth)) - apiRoute.Any("/datasources/uid/:uid/health", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealthWithUID)) + apiRoute.Any("/datasources/:id/health", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealth)) + apiRoute.Any("/datasources/uid/:uid/health", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.CheckDatasourceHealthWithUID)) // Folders apiRoute.Group("/folders", func(folderRoute routing.RouteRegister) { @@ -532,7 +532,7 @@ func (hs *HTTPServer) registerRoutes() { // metrics // DataSource w/ expressions - apiRoute.Post("/ds/query", authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.QueryMetricsV2)) + apiRoute.Post("/ds/query", requestmeta.SetSLOGroup(requestmeta.SLOGroupHighSlow), authorize(ac.EvalPermission(datasources.ActionQuery)), routing.Wrap(hs.QueryMetricsV2)) apiRoute.Group("/alerts", func(alertsRoute routing.RouteRegister) { alertsRoute.Post("/test", routing.Wrap(hs.AlertTest)) @@ -597,7 +597,7 @@ func (hs *HTTPServer) registerRoutes() { // Some channels may have info liveRoute.Get("/info/*", routing.Wrap(hs.Live.HandleInfoHTTP)) - }) + }, requestmeta.SetSLOGroup(requestmeta.SLOGroupNone)) // short urls apiRoute.Post("/short-urls", routing.Wrap(hs.createShortURL)) diff --git a/pkg/middleware/request_metrics.go b/pkg/middleware/request_metrics.go index bc3dba002e8..aed6d250751 100644 --- a/pkg/middleware/request_metrics.go +++ b/pkg/middleware/request_metrics.go @@ -45,6 +45,10 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR histogramLabels = append(histogramLabels, "team") } + if features.IsEnabled(featuremgmt.FlagHttpSLOLevels) { + histogramLabels = append(histogramLabels, "slo_group") + } + httpRequestDurationHistogram := prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "grafana", @@ -95,6 +99,10 @@ func RequestMetrics(features featuremgmt.FeatureToggles, cfg *setting.Cfg, promR labelValues = append(labelValues, rmd.Team) } + if features.IsEnabled(featuremgmt.FlagHttpSLOLevels) { + labelValues = append(labelValues, string(rmd.SLOGroup)) + } + // avoiding the sanitize functions for in the new instrumentation // since they dont make much sense. We should remove them later. histogram := httpRequestDurationHistogram. diff --git a/pkg/middleware/requestmeta/request_metadata.go b/pkg/middleware/requestmeta/request_metadata.go index a7ede04ff6d..e309483eb12 100644 --- a/pkg/middleware/requestmeta/request_metadata.go +++ b/pkg/middleware/requestmeta/request_metadata.go @@ -20,13 +20,38 @@ const ( StatusSourceDownstream StatusSource = "downstream" ) -type rMDContextKey struct{} - type RequestMetaData struct { Team string StatusSource StatusSource + SLOGroup SLOGroup } +type SLOGroup string + +const ( + // SLOGroupHighFast is the default slo group for handlers in Grafana + // Most handlers should respond quickly + SLOGroupHighFast SLOGroup = "high-fast" + + // SLOGroupHighMedium is used for handlers that might take some + // time to respond. + SLOGroupHighMedium SLOGroup = "high-medium" + + // SLOGroupHighSlow is used by handlers that proxies requests downstream. + // We expect an high successrate but without latency garantess + SLOGroupHighSlow SLOGroup = "high-slow" + + // SLOGroupLow should only be used for experimental features + // that might not be stable enough for our normal SLO + SLOGroupLow SLOGroup = "low" + + // SLOGroupNone means that errors are expect for this handler and + // it should not be included in the SLO. + SLOGroupNone SLOGroup = "none" +) + +type rMDContextKey struct{} + var requestMetaDataContextKey = rMDContextKey{} // SetupRequestMetadata injects defaul request metadata values @@ -96,5 +121,14 @@ func defaultRequestMetadata() RequestMetaData { return RequestMetaData{ Team: TeamCore, StatusSource: StatusSourceServer, + SLOGroup: SLOGroupHighFast, + } +} + +// SetSLOGroup creates an middleware that sets the SLOGroup for the request +func SetSLOGroup(lvl SLOGroup) web.Handler { + return func(w http.ResponseWriter, r *http.Request) { + v := GetRequestMetaData(r.Context()) + v.SLOGroup = lvl } } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index f28c6bf5c7c..a0d9724c10c 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -772,5 +772,13 @@ var ( Stage: FeatureStageExperimental, Owner: grafanaPluginsPlatformSquad, }, + { + Name: "httpSLOLevels", + Description: "Adds SLO level to http request metrics", + Stage: FeatureStageExperimental, + FrontendOnly: false, + Owner: hostedGrafanaTeam, + RequiresRestart: true, + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index e98f7f74cb1..c1fdd64001a 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -110,3 +110,4 @@ wargamesTesting,experimental,@grafana/hosted-grafana-team,false,false,false,fals alertingInsights,experimental,@grafana/alerting-squad,false,false,false,true externalCorePlugins,experimental,@grafana/plugins-platform-backend,false,false,false,false pluginsAPIMetrics,experimental,@grafana/plugins-platform-backend,false,false,false,true +httpSLOLevels,experimental,@grafana/hosted-grafana-team,false,false,true,false diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 3f6d18f77c1..14462a49dd7 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -450,4 +450,8 @@ const ( // FlagPluginsAPIMetrics // Sends metrics of public grafana packages usage by plugins FlagPluginsAPIMetrics = "pluginsAPIMetrics" + + // FlagHttpSLOLevels + // Adds SLO level to http request metrics + FlagHttpSLOLevels = "httpSLOLevels" )