Prometheus: Fix issue where TCP connections not being reused when querying from Grafana alerting (#40349)

Fixes #40366

Co-authored-by: Dave Henderson <dave.henderson@grafana.com>
Co-authored-by: Marcus Efraimsson <marcus.efraimsson@gmail.com>

pkg/tsdb/prometheus/prometheus.go

@@ -36,17 +36,15 @@ var (
 )
 
 type Service struct {
-	httpClientProvider httpclient.Provider
 	intervalCalculator intervalv2.Calculator
 	im                 instancemgmt.InstanceManager
 }
 
 func ProvideService(httpClientProvider httpclient.Provider, backendPluginManager backendplugin.Manager) (*Service, error) {
 	plog.Debug("initializing")
-	im := datasource.NewInstanceManager(newInstanceSettings())
+	im := datasource.NewInstanceManager(newInstanceSettings(httpClientProvider))
 
 	s := &Service{
-		httpClientProvider: httpClientProvider,
 		intervalCalculator: intervalv2.NewCalculator(),
 		im:                 im,
 	}
@@ -62,7 +60,7 @@ func ProvideService(httpClientProvider httpclient.Provider, backendPluginManager
 	return s, nil
 }
 
-func newInstanceSettings() datasource.InstanceFactoryFunc {
+func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
 	return func(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
 		defaultHttpMethod := http.MethodPost
 		jsonData := map[string]interface{}{}
@@ -99,13 +97,19 @@ func newInstanceSettings() datasource.InstanceFactoryFunc {
 			}
 		}
 
+		client, err := createClient(settings.URL, httpCliOpts, httpClientProvider)
+		if err != nil {
+			return nil, err
+		}
+
 		mdl := DatasourceInfo{
-			ID:             settings.ID,
-			URL:            settings.URL,
-			HTTPClientOpts: httpCliOpts,
-			HTTPMethod:     httpMethod,
-			TimeInterval:   timeInterval,
+			ID:           settings.ID,
+			URL:          settings.URL,
+			HTTPMethod:   httpMethod,
+			TimeInterval: timeInterval,
+			promClient:   client,
 		}
+
 		return mdl, nil
 	}
 }
@@ -120,10 +124,8 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 	if err != nil {
 		return nil, err
 	}
-	client, err := getClient(dsInfo, s)
-	if err != nil {
-		return nil, err
-	}
+
+	client := dsInfo.promClient
 
 	result := backend.QueryDataResponse{
 		Responses: backend.Responses{},
@@ -190,24 +192,17 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 	return &result, nil
 }
 
-func getClient(dsInfo *DatasourceInfo, s *Service) (apiv1.API, error) {
-	opts := &sdkhttpclient.Options{
-		Timeouts:  dsInfo.HTTPClientOpts.Timeouts,
-		TLS:       dsInfo.HTTPClientOpts.TLS,
-		BasicAuth: dsInfo.HTTPClientOpts.BasicAuth,
-		Headers:   dsInfo.HTTPClientOpts.Headers,
-	}
-
+func createClient(url string, httpOpts sdkhttpclient.Options, clientProvider httpclient.Provider) (apiv1.API, error) {
 	customMiddlewares := customQueryParametersMiddleware(plog)
-	opts.Middlewares = []sdkhttpclient.Middleware{customMiddlewares}
+	httpOpts.Middlewares = []sdkhttpclient.Middleware{customMiddlewares}
 
-	roundTripper, err := s.httpClientProvider.GetTransport(*opts)
+	roundTripper, err := clientProvider.GetTransport(httpOpts)
 	if err != nil {
 		return nil, err
 	}
 
 	cfg := api.Config{
-		Address:      dsInfo.URL,
+		Address:      url,
 		RoundTripper: roundTripper,
 	}
 

pkg/tsdb/prometheus/types.go

@@ -3,15 +3,16 @@ package prometheus
 import (
 	"time"
 
-	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 )
 
 type DatasourceInfo struct {
-	ID             int64
-	HTTPClientOpts sdkhttpclient.Options
-	URL            string
-	HTTPMethod     string
-	TimeInterval   string
+	ID           int64
+	URL          string
+	HTTPMethod   string
+	TimeInterval string
+
+	promClient apiv1.API
 }
 
 type PrometheusQuery struct {