Instrumentation: Fix HTTP request instrumentation of authentication failures (#44234)

Moves the request tracing middleware earlier in the chain, just after the tracing middleware 
and before the log middleware. With these changes we'll be able to track 
authentication/authorization status failures that currently exits early and don't execute the 
request tracing middleware. In addition, there might be some other routes now being tracked 
with this that we didn't do before.

Fixes #39590

pkg/api/http_server.go

@@ -432,6 +432,7 @@ func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() {
 	m := hs.web
 
 	m.Use(middleware.RequestTracing(hs.tracer))
+	m.Use(middleware.RequestMetrics(hs.Features))
 
 	m.Use(middleware.Logger(hs.Cfg))
 

pkg/api/routing/route_register.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/grafana/grafana/pkg/middleware"
-	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/web"
 )
 
@@ -52,8 +51,8 @@ type RouteRegister interface {
 
 type RegisterNamedMiddleware func(name string) web.Handler
 
-func ProvideRegister(features featuremgmt.FeatureToggles) *RouteRegisterImpl {
-	return NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(features))
+func ProvideRegister() *RouteRegisterImpl {
+	return NewRouteRegister(middleware.ProvideRouteOperationName)
 }
 
 // NewRouteRegister creates a new RouteRegister with all middlewares sent as params

pkg/middleware/request_metrics.go

@@ -45,15 +45,26 @@ func init() {
 }
 
 // RequestMetrics is a middleware handler that instruments the request.
-func RequestMetrics(features featuremgmt.FeatureToggles) func(handler string) web.Handler {
-	return func(handler string) web.Handler {
+func RequestMetrics(features featuremgmt.FeatureToggles) web.Handler {
 	return func(res http.ResponseWriter, req *http.Request, c *web.Context) {
+		if strings.HasPrefix(c.Req.URL.Path, "/public/") || c.Req.URL.Path == "robots.txt" || c.Req.URL.Path == "/metrics" {
+			c.Next()
+			return
+		}
+
 		rw := res.(web.ResponseWriter)
 		now := time.Now()
 		httpRequestsInFlight.Inc()
 		defer httpRequestsInFlight.Dec()
+		c.Map(c.Req)
 		c.Next()
 
+		handler := "unknown"
+
+		if routeOperation, exists := RouteOperationNameFromContext(c.Req.Context()); exists {
+			handler = routeOperation
+		}
+
 		status := rw.Status()
 
 		code := sanitizeCode(status)
@@ -68,7 +79,7 @@ func RequestMetrics(features featuremgmt.FeatureToggles) func(handler string) we
 			// avoiding the sanitize functions for in the new instrumentation
 			// since they dont make much sense. We should remove them later.
 			histogram := httpRequestDurationHistogram.
-					WithLabelValues(handler, strconv.Itoa(rw.Status()), req.Method)
+				WithLabelValues(handler, code, req.Method)
 			if traceID, ok := cw.ExtractSampledTraceID(c.Req.Context()); ok {
 				// Need to type-convert the Observer to an
 				// ExemplarObserver. This will always work for a
@@ -91,7 +102,6 @@ func RequestMetrics(features featuremgmt.FeatureToggles) func(handler string) we
 		}
 	}
 }
-}
 
 func countApiRequests(status int) {
 	switch status {