HTTP Client Metrics: Add datasource_type to datasource metrics (#72130)

Metrics: Add datasource_type to datasource metrics

pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go

@@ -18,7 +18,7 @@ var (
 			Name:      "datasource_request_total",
 			Help:      "A counter for outgoing requests for a data source",
 		},
-		[]string{"datasource", "code", "method"},
+		[]string{"datasource", "datasource_type", "code", "method"},
 	)
 
 	datasourceRequestHistogram = promauto.NewHistogramVec(
@@ -27,7 +27,7 @@ var (
 			Name:      "datasource_request_duration_seconds",
 			Help:      "histogram of durations of outgoing data source requests sent from Grafana",
 			Buckets:   []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100},
-		}, []string{"datasource", "code", "method"},
+		}, []string{"datasource", "datasource_type", "code", "method"},
 	)
 
 	datasourceResponseHistogram = promauto.NewHistogramVec(
@@ -36,7 +36,7 @@ var (
 			Name:      "datasource_response_size_bytes",
 			Help:      "histogram of data source response sizes returned to Grafana",
 			Buckets:   []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576},
-		}, []string{"datasource"},
+		}, []string{"datasource", "datasource_type"},
 	)
 
 	datasourceRequestsInFlight = promauto.NewGaugeVec(
@@ -45,7 +45,7 @@ var (
 			Name:      "datasource_request_in_flight",
 			Help:      "A gauge of outgoing data source requests currently being sent by Grafana",
 		},
-		[]string{"datasource"},
+		[]string{"datasource", "datasource_type"},
 	)
 )
 
@@ -71,18 +71,29 @@ func DataSourceMetricsMiddleware() sdkhttpclient.Middleware {
 			return next
 		}
 
-		datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName}
+		datasourceType, exists := opts.Labels["datasource_type"]
+		if !exists {
+			return next
+		}
+		datasourceLabelType, err := metricutil.SanitizeLabelName(datasourceType)
+		// if the datasource type cannot be turned into a prometheus
+		// label we will skip instrumenting these metrics.
+		if err != nil {
+			return next
+		}
+
+		labels := prometheus.Labels{"datasource": datasourceLabelName, "datasource_type": datasourceLabelType}
 
-		return executeMiddlewareFunc(next, datasourceLabel)
+		return executeMiddlewareFunc(next, labels)
 	})
 }
 
-func executeMiddleware(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
+func executeMiddleware(next http.RoundTripper, labels prometheus.Labels) http.RoundTripper {
 	return sdkhttpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
-		requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel)
-		requestHistogram := datasourceRequestHistogram.MustCurryWith(datasourceLabel)
-		requestInFlight := datasourceRequestsInFlight.With(datasourceLabel)
-		responseSizeHistogram := datasourceResponseHistogram.With(datasourceLabel)
+		requestCounter := datasourceRequestCounter.MustCurryWith(labels)
+		requestHistogram := datasourceRequestHistogram.MustCurryWith(labels)
+		requestInFlight := datasourceRequestsInFlight.With(labels)
+		responseSizeHistogram := datasourceResponseHistogram.With(labels)
 
 		res, err := promhttp.InstrumentRoundTripperDuration(requestHistogram,
 			promhttp.InstrumentRoundTripperCounter(requestCounter,

pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go

@@ -89,11 +89,11 @@ func TestDataSourceMetricsMiddleware(t *testing.T) {
 	t.Run("With datasource name label options set should execute middleware", func(t *testing.T) {
 		origExecuteMiddlewareFunc := executeMiddlewareFunc
 		executeMiddlewareCalled := false
-		datasourceLabels := prometheus.Labels{}
+		labels := prometheus.Labels{}
 		middlewareCalled := false
 		executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
 			executeMiddlewareCalled = true
-			datasourceLabels = datasourceLabel
+			labels = datasourceLabel
 			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
 				middlewareCalled = true
 				return next.RoundTrip(r)
@@ -106,7 +106,7 @@ func TestDataSourceMetricsMiddleware(t *testing.T) {
 		ctx := &testContext{}
 		finalRoundTripper := ctx.createRoundTripper("finalrt")
 		mw := DataSourceMetricsMiddleware()
-		rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123"}}, finalRoundTripper)
+		rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123", "datasource_type": "prometheus"}}, finalRoundTripper)
 		require.NotNil(t, rt)
 		middlewareName, ok := mw.(httpclient.MiddlewareName)
 		require.True(t, ok)
@@ -123,8 +123,9 @@ func TestDataSourceMetricsMiddleware(t *testing.T) {
 		require.Len(t, ctx.callChain, 1)
 		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
 		require.True(t, executeMiddlewareCalled)
-		require.Len(t, datasourceLabels, 1)
-		require.Equal(t, "My_Data_Source_123", datasourceLabels["datasource"])
+		require.Len(t, labels, 2)
+		require.Equal(t, "My_Data_Source_123", labels["datasource"])
+		require.Equal(t, "prometheus", labels["datasource_type"])
 		require.True(t, middlewareCalled)
 	})
 }