Datasource: Shared HTTP client provider for core backend data sources and any data source using the data source proxy (#33439)

Uses new httpclient package from grafana-plugin-sdk-go introduced 
via grafana/grafana-plugin-sdk-go#328. 
Replaces the GetHTTPClient, GetTransport, GetTLSConfig methods defined 
on DataSource model.
Longer-term the goal is to migrate core HTTP backend data sources to use the 
SDK contracts and using httpclient.Provider for creating HTTP clients and such.

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

conf/defaults.ini

@@ -158,6 +158,9 @@ expect_continue_timeout_seconds = 1
 # The maximum number of idle connections that Grafana will keep alive.
 max_idle_connections = 100
 
+# The maximum number of idle connections per host that Grafana will keep alive.
+max_idle_connections_per_host = 2
+
 # How many seconds the data proxy keeps an idle connection open before timing out.
 idle_conn_timeout_seconds = 90
 

conf/sample.ini

@@ -164,6 +164,9 @@
 # The maximum number of idle connections that Grafana will keep alive.
 ;max_idle_connections = 100
 
+# The maximum number of idle connections per host that Grafana will keep alive.
+;max_idle_connections_per_host = 2
+
 # How many seconds the data proxy keeps an idle connection open before timing out.
 ;idle_conn_timeout_seconds = 90
 

devenv/docker/blocks/slow_proxy/.env

@@ -0,0 +1,2 @@
+ORIGIN_SERVER=http://localhost:9090/
+SLEEP_DURATION=60s

devenv/docker/blocks/slow_proxy/docker-compose.yaml

@@ -4,4 +4,5 @@
     ports:
       - "3011:3011"
     environment:
-      ORIGIN_SERVER: "http://localhost:9090/"
+      ORIGIN_SERVER: ${ORIGIN_SERVER}
+      SLEEP_DURATION: ${SLEEP_DURATION}

devenv/docker/blocks/slow_proxy/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
@@ -16,13 +15,21 @@ func main() {
 		origin = "http://localhost:9090/"
 	}
 
-	sleep := time.Minute
+	sleepDurationStr := os.Getenv("SLEEP_DURATION")
+	if sleepDurationStr == "" {
+		sleepDurationStr = "60s"
+	}
+
+	sleep, err := time.ParseDuration(sleepDurationStr)
+	if err != nil {
+		log.Fatalf("failed to parse SLEEP_DURATION: %v", err)
+	}
 
 	originURL, _ := url.Parse(origin)
 	proxy := httputil.NewSingleHostReverseProxy(originURL)
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		fmt.Printf("sleeping for %s then proxying request: %s", sleep.String(), r.RequestURI)
+		log.Printf("sleeping for %s then proxying request: url '%s', headers: '%v'", sleep.String(), r.RequestURI, r.Header)
 		<-time.After(sleep)
 		proxy.ServeHTTP(w, r)
 	})

docs/sources/administration/configuration.md

@@ -418,6 +418,10 @@ The length of time that Grafana will wait for a datasource’s first response he
 
 The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation.
 
+### max_idle_connections_per_host
+
+The maximum number of idle connections per host that Grafana will maintain. Default is `2`. For more details check the [Transport.MaxIdleConnsPerHost](https://golang.org/pkg/net/http/#Transport.MaxIdleConnsPerHost) documentation.
+
 ### idle_conn_timeout_seconds
 
 The length of time that Grafana maintains idle connections before closing them. Default is `90` seconds. For more details check the [Transport.IdleConnTimeout](https://golang.org/pkg/net/http/#Transport.IdleConnTimeout) documentation.

pkg/api/pluginproxy/ds_proxy.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/api/datasource"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	glog "github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -30,13 +31,14 @@ var (
 )
 
 type DataSourceProxy struct {
-	ds        *models.DataSource
+	ds             *models.DataSource
-	ctx       *models.ReqContext
+	ctx            *models.ReqContext
-	targetUrl *url.URL
+	targetUrl      *url.URL
-	proxyPath string
+	proxyPath      string
-	route     *plugins.AppPluginRoute
+	route          *plugins.AppPluginRoute
-	plugin    *plugins.DataSourcePlugin
+	plugin         *plugins.DataSourcePlugin
-	cfg       *setting.Cfg
+	cfg            *setting.Cfg
+	clientProvider httpclient.Provider
 }
 
 type handleResponseTransport struct {
@@ -69,19 +71,20 @@ func (lw *logWrapper) Write(p []byte) (n int, err error) {
 
 // NewDataSourceProxy creates a new Datasource proxy
 func NewDataSourceProxy(ds *models.DataSource, plugin *plugins.DataSourcePlugin, ctx *models.ReqContext,
-	proxyPath string, cfg *setting.Cfg) (*DataSourceProxy, error) {
+	proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider) (*DataSourceProxy, error) {
 	targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
 	if err != nil {
 		return nil, err
 	}
 
 	return &DataSourceProxy{
-		ds:        ds,
+		ds:             ds,
-		plugin:    plugin,
+		plugin:         plugin,
-		ctx:       ctx,
+		ctx:            ctx,
-		proxyPath: proxyPath,
+		proxyPath:      proxyPath,
-		targetUrl: targetURL,
+		targetUrl:      targetURL,
-		cfg:       cfg,
+		cfg:            cfg,
+		clientProvider: clientProvider,
 	}, nil
 }
 
@@ -101,7 +104,7 @@ func (proxy *DataSourceProxy) HandleRequest() {
 	proxyErrorLogger := logger.New("userId", proxy.ctx.UserId, "orgId", proxy.ctx.OrgId, "uname", proxy.ctx.Login,
 		"path", proxy.ctx.Req.URL.Path, "remote_addr", proxy.ctx.RemoteAddr(), "referer", proxy.ctx.Req.Referer())
 
-	transport, err := proxy.ds.GetHttpTransport()
+	transport, err := proxy.ds.GetHTTPTransport(proxy.clientProvider)
 	if err != nil {
 		proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
 		return

pkg/api/pluginproxy/ds_proxy_test.go

@@ -12,23 +12,24 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/api/datasource"
+	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/securejsondata"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/login/social"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
 	"golang.org/x/oauth2"
 	macaron "gopkg.in/macaron.v1"
-
-	"github.com/grafana/grafana/pkg/bus"
-	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/login/social"
-	"github.com/grafana/grafana/pkg/setting"
-	"github.com/grafana/grafana/pkg/util"
 )
 
 func TestDataSourceProxy_routeRule(t *testing.T) {
+	httpClientProvider := httpclient.NewProvider()
+
 	t.Run("Plugin with routes", func(t *testing.T) {
 		plugin := &plugins.DataSourcePlugin{
 			Routes: []*plugins.AppPluginRoute{
@@ -113,7 +114,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 		t.Run("When matching route path", func(t *testing.T) {
 			ctx, req := setUp()
-			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg)
+			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider)
 			require.NoError(t, err)
 			proxy.route = plugin.Routes[0]
 			ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@@ -124,7 +125,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 		t.Run("When matching route path and has dynamic url", func(t *testing.T) {
 			ctx, req := setUp()
-			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg)
+			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg, httpClientProvider)
 			require.NoError(t, err)
 			proxy.route = plugin.Routes[3]
 			ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@@ -135,7 +136,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 		t.Run("When matching route path with no url", func(t *testing.T) {
 			ctx, req := setUp()
-			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg)
+			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpClientProvider)
 			require.NoError(t, err)
 			proxy.route = plugin.Routes[4]
 			ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@@ -145,7 +146,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 		t.Run("When matching route path and has dynamic body", func(t *testing.T) {
 			ctx, req := setUp()
-			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg)
+			proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg, httpClientProvider)
 			require.NoError(t, err)
 			proxy.route = plugin.Routes[5]
 			ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg)
@@ -158,7 +159,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 		t.Run("Validating request", func(t *testing.T) {
 			t.Run("plugin route with valid role", func(t *testing.T) {
 				ctx, _ := setUp()
-				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg)
+				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider)
 				require.NoError(t, err)
 				err = proxy.validateRequest()
 				require.NoError(t, err)
@@ -166,7 +167,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 			t.Run("plugin route with admin role and user is editor", func(t *testing.T) {
 				ctx, _ := setUp()
-				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg)
+				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider)
 				require.NoError(t, err)
 				err = proxy.validateRequest()
 				require.Error(t, err)
@@ -175,7 +176,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 			t.Run("plugin route with admin role and user is admin", func(t *testing.T) {
 				ctx, _ := setUp()
 				ctx.SignedInUser.OrgRole = models.ROLE_ADMIN
-				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg)
+				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider)
 				require.NoError(t, err)
 				err = proxy.validateRequest()
 				require.NoError(t, err)
@@ -257,7 +258,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 				cfg := &setting.Cfg{}
 
-				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg)
+				proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider)
 				require.NoError(t, err)
 				ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg)
 
@@ -272,7 +273,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 					req, err := http.NewRequest("GET", "http://localhost/asd", nil)
 					require.NoError(t, err)
 					client = newFakeHTTPClient(t, json2)
-					proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg)
+					proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg, httpClientProvider)
 					require.NoError(t, err)
 					ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[1], proxy.ds, cfg)
 
@@ -288,7 +289,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 						require.NoError(t, err)
 
 						client = newFakeHTTPClient(t, []byte{})
-						proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg)
+						proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider)
 						require.NoError(t, err)
 						ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg)
 
@@ -304,17 +305,11 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 	})
 
 	t.Run("When proxying graphite", func(t *testing.T) {
-		origBuildVer := setting.BuildVersion
-		t.Cleanup(func() {
-			setting.BuildVersion = origBuildVer
-		})
-		setting.BuildVersion = "5.3.0"
-
 		plugin := &plugins.DataSourcePlugin{}
 		ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE}
 		ctx := &models.ReqContext{}
 
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider)
 		require.NoError(t, err)
 		req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
 		require.NoError(t, err)
@@ -324,7 +319,6 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 		t.Run("Can translate request URL and path", func(t *testing.T) {
 			assert.Equal(t, "graphite:8080", req.URL.Host)
 			assert.Equal(t, "/render", req.URL.Path)
-			assert.Equal(t, "Grafana/5.3.0", req.Header.Get("User-Agent"))
 		})
 	})
 
@@ -340,7 +334,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 		}
 
 		ctx := &models.ReqContext{}
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@@ -363,7 +357,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 		}
 
 		ctx := &models.ReqContext{}
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		requestURL, err := url.Parse("http://grafana.com/sub")
@@ -390,7 +384,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 		}
 
 		ctx := &models.ReqContext{}
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		requestURL, err := url.Parse("http://grafana.com/sub")
@@ -411,7 +405,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 			Url:  "http://host/root/",
 		}
 		ctx := &models.ReqContext{}
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 		req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
 		req.Header.Set("Origin", "grafana.com")
@@ -472,7 +466,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 				Req: macaron.Request{Request: req},
 			},
 		}
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 		req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
 		require.NoError(t, err)
@@ -545,6 +539,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) {
 
 // test DataSourceProxy request handling.
 func TestDataSourceProxy_requestHandling(t *testing.T) {
+	httpClientProvider := httpclient.NewProvider()
 	var writeErr error
 
 	plugin := &plugins.DataSourcePlugin{}
@@ -605,7 +600,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
 
 	t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) {
 		ctx, ds := setUp(t)
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		proxy.HandleRequest()
@@ -620,7 +615,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
 				"Set-Cookie": "important_cookie=important_value",
 			},
 		})
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		proxy.HandleRequest()
@@ -639,7 +634,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
 				t.Log("Wrote 401 response")
 			},
 		})
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		proxy.HandleRequest()
@@ -661,7 +656,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) {
 		})
 
 		ctx.Req.Request = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil)
-		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{})
+		proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider)
 		require.NoError(t, err)
 
 		proxy.HandleRequest()
@@ -685,7 +680,7 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) {
 	}
 	cfg := setting.Cfg{}
 	plugin := plugins.DataSourcePlugin{}
-	_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
+	_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
 	require.Error(t, err)
 	assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`))
 }
@@ -704,7 +699,7 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) {
 	cfg := setting.Cfg{}
 	plugin := plugins.DataSourcePlugin{}
 
-	_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
+	_, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
 
 	require.NoError(t, err)
 }
@@ -744,7 +739,7 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) {
 				Url:  tc.url,
 			}
 
-			p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg)
+			p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider())
 			if tc.err == nil {
 				require.NoError(t, err)
 				assert.Equal(t, &url.URL{
@@ -782,7 +777,7 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett
 		Url:  "http://host/root/",
 	}
 
-	proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg)
+	proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpclient.NewProvider())
 	require.NoError(t, err)
 	req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
 	require.NoError(t, err)
@@ -840,6 +835,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri
 
 	test := &testCase{
 		datasource: &models.DataSource{
+			Id:       1,
 			Type:     dsType,
 			JsonData: simplejson.New(),
 		},
@@ -892,7 +888,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri
 func runDatasourceAuthTest(t *testing.T, test *testCase) {
 	plugin := &plugins.DataSourcePlugin{}
 	ctx := &models.ReqContext{}
-	proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{})
+	proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{}, httpclient.NewProvider())
 	require.NoError(t, err)
 
 	req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
@@ -933,7 +929,7 @@ func Test_PathCheck(t *testing.T) {
 		return ctx, req
 	}
 	ctx, _ := setUp()
-	proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{})
+	proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{}, httpclient.NewProvider())
 	require.NoError(t, err)
 
 	require.Nil(t, proxy.validateRequest())

pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go

@@ -0,0 +1,105 @@
+package httpclientprovider
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var datasourceRequestCounter = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "grafana",
+		Name:      "datasource_request_total",
+		Help:      "A counter for outgoing requests for a datasource",
+	},
+	[]string{"datasource", "code", "method"},
+)
+
+var datasourceRequestSummary = prometheus.NewSummaryVec(
+	prometheus.SummaryOpts{
+		Namespace:  "grafana",
+		Name:       "datasource_request_duration_seconds",
+		Help:       "summary of outgoing datasource requests sent from Grafana",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	}, []string{"datasource", "code", "method"},
+)
+
+var datasourceResponseSummary = prometheus.NewSummaryVec(
+	prometheus.SummaryOpts{
+		Namespace:  "grafana",
+		Name:       "datasource_response_size_bytes",
+		Help:       "summary of datasource response sizes returned to Grafana",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	}, []string{"datasource"},
+)
+
+var datasourceRequestsInFlight = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: "grafana",
+		Name:      "datasource_request_in_flight",
+		Help:      "A gauge of outgoing datasource requests currently being sent by Grafana",
+	},
+	[]string{"datasource"},
+)
+
+func init() {
+	prometheus.MustRegister(datasourceRequestSummary,
+		datasourceRequestCounter,
+		datasourceRequestsInFlight,
+		datasourceResponseSummary)
+}
+
+const DataSourceMetricsMiddlewareName = "metrics"
+
+var executeMiddlewareFunc = executeMiddleware
+
+func DataSourceMetricsMiddleware() httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(DataSourceMetricsMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		if opts.Labels == nil {
+			return next
+		}
+
+		datasourceName, exists := opts.Labels["datasource_name"]
+		if !exists {
+			return next
+		}
+
+		datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName)
+		// if the datasource named cannot be turned into a prometheus
+		// label we will skip instrumenting these metrics.
+		if err != nil {
+			return next
+		}
+
+		datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName}
+
+		return executeMiddlewareFunc(next, datasourceLabel)
+	})
+}
+
+func executeMiddleware(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
+	return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+		requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel)
+		requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel)
+		requestInFlight := datasourceRequestsInFlight.With(datasourceLabel)
+		responseSizeSummary := datasourceResponseSummary.With(datasourceLabel)
+
+		res, err := promhttp.InstrumentRoundTripperDuration(requestSummary,
+			promhttp.InstrumentRoundTripperCounter(requestCounter,
+				promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))).
+			RoundTrip(r)
+		if err != nil {
+			return nil, err
+		}
+		// we avoid measuring contentlength less than zero because it indicates
+		// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
+		if res != nil && res.ContentLength > 0 {
+			responseSizeSummary.Observe(float64(res.ContentLength))
+		}
+
+		return res, nil
+	})
+}

pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go

@@ -0,0 +1,130 @@
+package httpclientprovider
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDataSourceMetricsMiddleware(t *testing.T) {
+	t.Run("Without label options set should return next http.RoundTripper", func(t *testing.T) {
+		origExecuteMiddlewareFunc := executeMiddlewareFunc
+		executeMiddlewareCalled := false
+		middlewareCalled := false
+		executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
+			executeMiddlewareCalled = true
+			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				middlewareCalled = true
+				return next.RoundTrip(r)
+			})
+		}
+		t.Cleanup(func() {
+			executeMiddlewareFunc = origExecuteMiddlewareFunc
+		})
+
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("finalrt")
+		mw := DataSourceMetricsMiddleware()
+		rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
+		require.False(t, executeMiddlewareCalled)
+		require.False(t, middlewareCalled)
+	})
+
+	t.Run("Without data source name label options set should return next http.RoundTripper", func(t *testing.T) {
+		origExecuteMiddlewareFunc := executeMiddlewareFunc
+		executeMiddlewareCalled := false
+		middlewareCalled := false
+		executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
+			executeMiddlewareCalled = true
+			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				middlewareCalled = true
+				return next.RoundTrip(r)
+			})
+		}
+		t.Cleanup(func() {
+			executeMiddlewareFunc = origExecuteMiddlewareFunc
+		})
+
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("finalrt")
+		mw := DataSourceMetricsMiddleware()
+		rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"test": "test"}}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
+		require.False(t, executeMiddlewareCalled)
+		require.False(t, middlewareCalled)
+	})
+
+	t.Run("With datasource name label options set should execute middleware", func(t *testing.T) {
+		origExecuteMiddlewareFunc := executeMiddlewareFunc
+		executeMiddlewareCalled := false
+		datasourceLabels := prometheus.Labels{}
+		middlewareCalled := false
+		executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper {
+			executeMiddlewareCalled = true
+			datasourceLabels = datasourceLabel
+			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				middlewareCalled = true
+				return next.RoundTrip(r)
+			})
+		}
+		t.Cleanup(func() {
+			executeMiddlewareFunc = origExecuteMiddlewareFunc
+		})
+
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("finalrt")
+		mw := DataSourceMetricsMiddleware()
+		rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123"}}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
+		require.True(t, executeMiddlewareCalled)
+		require.Len(t, datasourceLabels, 1)
+		require.Equal(t, "My_Data_Source_123", datasourceLabels["datasource"])
+		require.True(t, middlewareCalled)
+	})
+}

pkg/infra/httpclient/httpclientprovider/http_client_provider.go

@@ -0,0 +1,30 @@
+package httpclientprovider
+
+import (
+	"fmt"
+
+	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+var newProviderFunc = sdkhttpclient.NewProvider
+
+// New creates a new HTTP client provider with pre-configured middlewares.
+func New(cfg *setting.Cfg) httpclient.Provider {
+	userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion)
+	middlewares := []sdkhttpclient.Middleware{
+		DataSourceMetricsMiddleware(),
+		SetUserAgentMiddleware(userAgent),
+		sdkhttpclient.BasicAuthenticationMiddleware(),
+		sdkhttpclient.CustomHeadersMiddleware(),
+	}
+
+	if cfg.SigV4AuthEnabled {
+		middlewares = append(middlewares, SigV4Middleware())
+	}
+
+	return newProviderFunc(sdkhttpclient.ProviderOptions{
+		Middlewares: middlewares,
+	})
+}

pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go

@@ -0,0 +1,52 @@
+package httpclientprovider
+
+import (
+	"testing"
+
+	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPClientProvider(t *testing.T) {
+	t.Run("When creating new provider and SigV4 is disabled should apply expected middleware", func(t *testing.T) {
+		origNewProviderFunc := newProviderFunc
+		providerOpts := []sdkhttpclient.ProviderOptions{}
+		newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider {
+			providerOpts = opts
+			return nil
+		}
+		t.Cleanup(func() {
+			newProviderFunc = origNewProviderFunc
+		})
+		_ = New(&setting.Cfg{SigV4AuthEnabled: false})
+		require.Len(t, providerOpts, 1)
+		o := providerOpts[0]
+		require.Len(t, o.Middlewares, 4)
+		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
+	})
+
+	t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) {
+		origNewProviderFunc := newProviderFunc
+		providerOpts := []sdkhttpclient.ProviderOptions{}
+		newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider {
+			providerOpts = opts
+			return nil
+		}
+		t.Cleanup(func() {
+			newProviderFunc = origNewProviderFunc
+		})
+		_ = New(&setting.Cfg{SigV4AuthEnabled: true})
+		require.Len(t, providerOpts, 1)
+		o := providerOpts[0]
+		require.Len(t, o.Middlewares, 5)
+		require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
+		require.Equal(t, SigV4MiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
+	})
+}

pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go

@@ -0,0 +1,36 @@
+package httpclientprovider
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+)
+
+// SigV4MiddlewareName the middleware name used by SigV4Middleware.
+const SigV4MiddlewareName = "sigv4"
+
+var newSigV4Func = sigv4.New
+
+// SigV4Middleware applies AWS Signature Version 4 request signing for the outgoing request.
+func SigV4Middleware() httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(SigV4MiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		if opts.SigV4 == nil {
+			return next
+		}
+
+		return newSigV4Func(
+			&sigv4.Config{
+				Service:       opts.SigV4.Service,
+				AccessKey:     opts.SigV4.AccessKey,
+				SecretKey:     opts.SigV4.SecretKey,
+				Region:        opts.SigV4.Region,
+				AssumeRoleARN: opts.SigV4.AssumeRoleARN,
+				AuthType:      opts.SigV4.AuthType,
+				ExternalID:    opts.SigV4.ExternalID,
+				Profile:       opts.SigV4.Profile,
+			},
+			next,
+		)
+	})
+}

pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go

@@ -0,0 +1,89 @@
+package httpclientprovider
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSigV4Middleware(t *testing.T) {
+	t.Run("Without sigv4 options set should return next http.RoundTripper", func(t *testing.T) {
+		origSigV4Func := newSigV4Func
+		newSigV4Called := false
+		middlewareCalled := false
+		newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper {
+			newSigV4Called = true
+			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				middlewareCalled = true
+				return next.RoundTrip(r)
+			})
+		}
+		t.Cleanup(func() {
+			newSigV4Func = origSigV4Func
+		})
+
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("finalrt")
+		mw := SigV4Middleware()
+		rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
+		require.False(t, newSigV4Called)
+		require.False(t, middlewareCalled)
+	})
+
+	t.Run("With sigv4 options set should call sigv4 http.RoundTripper", func(t *testing.T) {
+		origSigV4Func := newSigV4Func
+		newSigV4Called := false
+		middlewareCalled := false
+		newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper {
+			newSigV4Called = true
+			return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
+				middlewareCalled = true
+				return next.RoundTrip(r)
+			})
+		}
+		t.Cleanup(func() {
+			newSigV4Func = origSigV4Func
+		})
+
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("final")
+		mw := SigV4Middleware()
+		rt := mw.CreateMiddleware(httpclient.Options{SigV4: &httpclient.SigV4Config{}}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"final"}, ctx.callChain)
+
+		require.True(t, newSigV4Called)
+		require.True(t, middlewareCalled)
+	})
+}

pkg/infra/httpclient/httpclientprovider/testing.go

@@ -0,0 +1,18 @@
+package httpclientprovider
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+)
+
+type testContext struct {
+	callChain []string
+}
+
+func (c *testContext) createRoundTripper(name string) http.RoundTripper {
+	return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		c.callChain = append(c.callChain, name)
+		return &http.Response{StatusCode: http.StatusOK}, nil
+	})
+}

pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go

@@ -0,0 +1,27 @@
+package httpclientprovider
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+)
+
+// SetUserAgentMiddlewareName is the middleware name used by SetUserAgentMiddleware.
+const SetUserAgentMiddlewareName = "user-agent"
+
+// SetUserAgentMiddleware is middleware that sets the HTTP header User-Agent on the outgoing request.
+// If User-Agent already set, it will not be overridden by this middleware.
+func SetUserAgentMiddleware(userAgent string) httpclient.Middleware {
+	return httpclient.NamedMiddlewareFunc(SetUserAgentMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper {
+		if userAgent == "" {
+			return next
+		}
+
+		return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			if req.Header.Get("User-Agent") == "" {
+				req.Header.Set("User-Agent", userAgent)
+			}
+			return next.RoundTrip(req)
+		})
+	})
+}

pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go

@@ -0,0 +1,82 @@
+package httpclientprovider
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCustomHeadersMiddleware(t *testing.T) {
+	t.Run("Without user agent set should return next http.RoundTripper", func(t *testing.T) {
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("finalrt")
+		mw := SetUserAgentMiddleware("")
+		rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain)
+	})
+
+	t.Run("With user agent set should apply HTTP headers to the request", func(t *testing.T) {
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("final")
+		mw := SetUserAgentMiddleware("Grafana/8.0.0")
+		rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"final"}, ctx.callChain)
+
+		require.Equal(t, "Grafana/8.0.0", req.Header.Get("User-Agent"))
+	})
+
+	t.Run("With user agent set, but request already has User-Agent header set should not apply HTTP headers to the request", func(t *testing.T) {
+		ctx := &testContext{}
+		finalRoundTripper := ctx.createRoundTripper("final")
+		mw := SetUserAgentMiddleware("Grafana/8.0.0")
+		rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+		require.NotNil(t, rt)
+		middlewareName, ok := mw.(httpclient.MiddlewareName)
+		require.True(t, ok)
+		require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName())
+
+		req, err := http.NewRequest(http.MethodGet, "http://", nil)
+		require.NoError(t, err)
+		req.Header.Set("User-Agent", "ua")
+		res, err := rt.RoundTrip(req)
+		require.NoError(t, err)
+		require.NotNil(t, res)
+		if res.Body != nil {
+			require.NoError(t, res.Body.Close())
+		}
+		require.Len(t, ctx.callChain, 1)
+		require.ElementsMatch(t, []string{"final"}, ctx.callChain)
+
+		require.Equal(t, "ua", req.Header.Get("User-Agent"))
+	})
+}

pkg/infra/httpclient/provider.go

@@ -0,0 +1,32 @@
+package httpclient
+
+import (
+	"crypto/tls"
+	"net/http"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+)
+
+// Provider provides abilities to create http.Client, http.RoundTripper and tls.Config.
+type Provider interface {
+	// New creates a new http.Client given provided options.
+	New(opts ...httpclient.Options) (*http.Client, error)
+
+	// GetTransport creates a new http.RoundTripper given provided options.
+	GetTransport(opts ...httpclient.Options) (http.RoundTripper, error)
+
+	// GetTLSConfig creates a new tls.Config given provided options.
+	GetTLSConfig(opts ...httpclient.Options) (*tls.Config, error)
+}
+
+// NewProvider creates a new HTTP client provider.
+// Optionally provide ProviderOptions options that will be used as default if
+// not specified in Options argument to Provider.New, Provider.GetTransport and
+// Provider.GetTLSConfig.
+// If no middlewares are provided in opts the DefaultMiddlewares() will be used. If you
+// provide middlewares you have to manually add the DefaultMiddlewares() for it to be
+// enabled.
+// Note: Middlewares will be executed in the same order as provided.
+func NewProvider(opts ...httpclient.ProviderOptions) *httpclient.Provider {
+	return httpclient.NewProvider(opts...)
+}

pkg/models/datasource_cache.go

@@ -2,128 +2,17 @@ package models
 
 import (
 	"crypto/tls"
-	"crypto/x509"
-	"errors"
 	"fmt"
-	"net"
 	"net/http"
 	"sync"
 	"time"
 
-	"github.com/grafana/grafana-aws-sdk/pkg/sigv4"
+	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
-
+	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/setting"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-)
-
-var datasourceRequestCounter = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: "grafana",
-		Name:      "datasource_request_total",
-		Help:      "A counter for outgoing requests for a datasource",
-	},
-	[]string{"datasource", "code", "method"},
-)
-
-var datasourceRequestSummary = prometheus.NewSummaryVec(
-	prometheus.SummaryOpts{
-		Namespace:  "grafana",
-		Name:       "datasource_request_duration_seconds",
-		Help:       "summary of outgoing datasource requests sent from Grafana",
-		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-	}, []string{"datasource", "code", "method"},
-)
-
-var datasourceResponseSummary = prometheus.NewSummaryVec(
-	prometheus.SummaryOpts{
-		Namespace:  "grafana",
-		Name:       "datasource_response_size_bytes",
-		Help:       "summary of datasource response sizes returned to Grafana",
-		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-	}, []string{"datasource"},
-)
-
-var datasourceRequestsInFlight = prometheus.NewGaugeVec(
-	prometheus.GaugeOpts{
-		Namespace: "grafana",
-		Name:      "datasource_request_in_flight",
-		Help:      "A gauge of outgoing datasource requests currently being sent by Grafana",
-	},
-	[]string{"datasource"},
 )
 
-func init() {
-	prometheus.MustRegister(datasourceRequestSummary,
-		datasourceRequestCounter,
-		datasourceRequestsInFlight,
-		datasourceResponseSummary)
-}
-
-type proxyTransportCache struct {
-	cache map[int64]cachedTransport
-	sync.Mutex
-}
-
-// dataSourceTransport implements http.RoundTripper (https://golang.org/pkg/net/http/#RoundTripper)
-type dataSourceTransport struct {
-	datasourceName string
-	headers        map[string]string
-	transport      *http.Transport
-	next           http.RoundTripper
-}
-
-func instrumentRoundtrip(datasourceName string, next http.RoundTripper) promhttp.RoundTripperFunc {
-	return promhttp.RoundTripperFunc(func(r *http.Request) (*http.Response, error) {
-		datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName)
-		// if the datasource named cannot be turned into a prometheus
-		// label we will skip instrumenting these metrics.
-		if err != nil {
-			return next.RoundTrip(r)
-		}
-
-		datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName}
-
-		requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel)
-		requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel)
-		requestInFlight := datasourceRequestsInFlight.With(datasourceLabel)
-		responseSizeSummary := datasourceResponseSummary.With(datasourceLabel)
-
-		res, err := promhttp.InstrumentRoundTripperDuration(requestSummary,
-			promhttp.InstrumentRoundTripperCounter(requestCounter,
-				promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))).
-			RoundTrip(r)
-
-		// we avoid measuring contentlength less than zero because it indicates
-		// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
-		if res != nil && res.ContentLength > 0 {
-			responseSizeSummary.Observe(float64(res.ContentLength))
-		}
-
-		return res, err
-	})
-}
-
-// RoundTrip executes a single HTTP transaction, returning a Response for the provided Request.
-func (d *dataSourceTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	for key, value := range d.headers {
-		req.Header.Set(key, value)
-	}
-
-	return instrumentRoundtrip(d.datasourceName, d.next).RoundTrip(req)
-}
-
-type cachedTransport struct {
-	updated time.Time
-
-	*dataSourceTransport
-}
-
-var ptc = proxyTransportCache{
-	cache: make(map[int64]cachedTransport),
-}
-
 func (ds *DataSource) getTimeout() time.Duration {
 	timeout := 0
 	if ds.JsonData != nil {
@@ -135,8 +24,22 @@ func (ds *DataSource) getTimeout() time.Duration {
 	return time.Duration(timeout) * time.Second
 }
 
-func (ds *DataSource) GetHttpClient() (*http.Client, error) {
+type proxyTransportCache struct {
-	transport, err := ds.GetHttpTransport()
+	cache map[int64]cachedRoundTripper
+	sync.Mutex
+}
+
+type cachedRoundTripper struct {
+	updated      time.Time
+	roundTripper http.RoundTripper
+}
+
+var ptc = proxyTransportCache{
+	cache: make(map[int64]cachedRoundTripper),
+}
+
+func (ds *DataSource) GetHTTPClient(provider httpclient.Provider) (*http.Client, error) {
+	transport, err := ds.GetHTTPTransport(provider)
 	if err != nil {
 		return nil, err
 	}
@@ -147,79 +50,86 @@ func (ds *DataSource) GetHttpClient() (*http.Client, error) {
 	}, nil
 }
 
-// Creates a HTTP Transport middleware chain
+func (ds *DataSource) GetHTTPTransport(provider httpclient.Provider) (http.RoundTripper, error) {
-func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) {
 	ptc.Lock()
 	defer ptc.Unlock()
 
 	if t, present := ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) {
-		return t.dataSourceTransport, nil
+		return t.roundTripper, nil
 	}
 
-	tlsConfig, err := ds.GetTLSConfig()
+	rt, err := provider.GetTransport(ds.HTTPClientOptions())
 	if err != nil {
 		return nil, err
 	}
 
-	tlsConfig.Renegotiation = tls.RenegotiateFreelyAsClient
+	ptc.cache[ds.Id] = cachedRoundTripper{
-
+		roundTripper: rt,
-	// Create transport which adds all
+		updated:      ds.Updated,
-	customHeaders := ds.getCustomHeaders()
-	transport := &http.Transport{
-		TLSClientConfig: tlsConfig,
-		Proxy:           http.ProxyFromEnvironment,
-		Dial: (&net.Dialer{
-			Timeout:   ds.getTimeout(),
-			KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second,
-		}).Dial,
-		TLSHandshakeTimeout:   time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
-		ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
-		MaxIdleConns:          setting.DataProxyMaxIdleConns,
-		IdleConnTimeout:       time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,
 	}
 
-	// Set default next round tripper to the default transport
+	return rt, nil
-	next := http.RoundTripper(transport)
+}
 
-	// Add SigV4 middleware if enabled, which will then defer to the default transport
+func (ds *DataSource) HTTPClientOptions() sdkhttpclient.Options {
-	if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool() && setting.SigV4AuthEnabled {
+	tlsOptions := ds.TLSOptions()
-		next = ds.sigV4Middleware(transport)
+	opts := sdkhttpclient.Options{
+		Timeouts: &sdkhttpclient.TimeoutOptions{
+			Timeout:               ds.getTimeout(),
+			KeepAlive:             time.Duration(setting.DataProxyKeepAlive) * time.Second,
+			TLSHandshakeTimeout:   time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
+			ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
+			MaxIdleConns:          setting.DataProxyMaxIdleConns,
+			MaxIdleConnsPerHost:   setting.DataProxyMaxIdleConnsPerHost,
+			IdleConnTimeout:       time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,
+		},
+		Headers: getCustomHeaders(ds.JsonData, ds.DecryptedValues()),
+		Labels: map[string]string{
+			"datasource_name": ds.Name,
+			"datasource_uid":  ds.Uid,
+		},
+		TLS: &tlsOptions,
 	}
 
-	dsTransport := &dataSourceTransport{
+	if ds.JsonData != nil {
-		datasourceName: ds.Name,
+		opts.CustomOptions = ds.JsonData.MustMap()
-		headers:        customHeaders,
-		transport:      transport,
-		next:           next,
 	}
 
-	ptc.cache[ds.Id] = cachedTransport{
+	if ds.BasicAuth {
-		dataSourceTransport: dsTransport,
+		opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
-		updated:             ds.Updated,
+			User:     ds.BasicAuthUser,
+			Password: ds.DecryptedBasicAuthPassword(),
+		}
+	} else if ds.User != "" {
+		opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{
+			User:     ds.User,
+			Password: ds.DecryptedPassword(),
+		}
 	}
 
-	return dsTransport, nil
+	if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool(false) {
-}
+		opts.SigV4 = &sdkhttpclient.SigV4Config{
-
-func (ds *DataSource) sigV4Middleware(next http.RoundTripper) http.RoundTripper {
-	decrypted := ds.DecryptedValues()
-
-	return sigv4.New(
-		&sigv4.Config{
 			Service:       awsServiceNamespace(ds.Type),
-			AccessKey:     decrypted["sigV4AccessKey"],
-			SecretKey:     decrypted["sigV4SecretKey"],
 			Region:        ds.JsonData.Get("sigV4Region").MustString(),
 			AssumeRoleARN: ds.JsonData.Get("sigV4AssumeRoleArn").MustString(),
 			AuthType:      ds.JsonData.Get("sigV4AuthType").MustString(),
 			ExternalID:    ds.JsonData.Get("sigV4ExternalId").MustString(),
 			Profile:       ds.JsonData.Get("sigV4Profile").MustString(),
-		},
+		}
-		next,
+
-	)
+		if val, exists := ds.DecryptedValue("sigV4AccessKey"); exists {
+			opts.SigV4.AccessKey = val
+		}
+
+		if val, exists := ds.DecryptedValue("sigV4SecretKey"); exists {
+			opts.SigV4.SecretKey = val
+		}
+	}
+
+	return opts
 }
 
-func (ds *DataSource) GetTLSConfig() (*tls.Config, error) {
+func (ds *DataSource) TLSOptions() sdkhttpclient.TLSOptions {
 	var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool
 	var serverName string
 
@@ -230,55 +140,55 @@ func (ds *DataSource) GetTLSConfig() (*tls.Config, error) {
 		serverName = ds.JsonData.Get("serverName").MustString()
 	}
 
-	tlsConfig := &tls.Config{
+	opts := sdkhttpclient.TLSOptions{
 		InsecureSkipVerify: tlsSkipVerify,
 		ServerName:         serverName,
 	}
 
 	if tlsClientAuth || tlsAuthWithCACert {
-		decrypted := ds.SecureJsonData.Decrypt()
+		if tlsAuthWithCACert {
-		if tlsAuthWithCACert && len(decrypted["tlsCACert"]) > 0 {
+			if val, exists := ds.DecryptedValue("tlsCACert"); exists && len(val) > 0 {
-			caPool := x509.NewCertPool()
+				opts.CACertificate = val
-			ok := caPool.AppendCertsFromPEM([]byte(decrypted["tlsCACert"]))
-			if !ok {
-				return nil, errors.New("failed to parse TLS CA PEM certificate")
 			}
-			tlsConfig.RootCAs = caPool
 		}
 
 		if tlsClientAuth {
-			cert, err := tls.X509KeyPair([]byte(decrypted["tlsClientCert"]), []byte(decrypted["tlsClientKey"]))
+			if val, exists := ds.DecryptedValue("tlsClientCert"); exists && len(val) > 0 {
-			if err != nil {
+				opts.ClientCertificate = val
-				return nil, err
+			}
+			if val, exists := ds.DecryptedValue("tlsClientKey"); exists && len(val) > 0 {
+				opts.ClientKey = val
 			}
-			tlsConfig.Certificates = []tls.Certificate{cert}
 		}
 	}
 
-	return tlsConfig, nil
+	return opts
+}
+
+func (ds *DataSource) GetTLSConfig(httpClientProvider httpclient.Provider) (*tls.Config, error) {
+	return httpClientProvider.GetTLSConfig(ds.HTTPClientOptions())
 }
 
 // getCustomHeaders returns a map with all the to be set headers
 // The map key represents the HeaderName and the value represents this header's value
-func (ds *DataSource) getCustomHeaders() map[string]string {
+func getCustomHeaders(jsonData *simplejson.Json, decryptedValues map[string]string) map[string]string {
 	headers := make(map[string]string)
-	if ds.JsonData == nil {
+	if jsonData == nil {
 		return headers
 	}
 
-	decrypted := ds.SecureJsonData.Decrypt()
 	index := 1
 	for {
 		headerNameSuffix := fmt.Sprintf("httpHeaderName%d", index)
 		headerValueSuffix := fmt.Sprintf("httpHeaderValue%d", index)
 
-		key := ds.JsonData.Get(headerNameSuffix).MustString()
+		key := jsonData.Get(headerNameSuffix).MustString()
 		if key == "" {
 			// No (more) header values are available
 			break
 		}
 
-		if val, ok := decrypted[headerValueSuffix]; ok {
+		if val, ok := decryptedValues[headerValueSuffix]; ok {
 			headers[key] = val
 		}
 		index++

pkg/models/datasource_cache_test.go

@@ -8,17 +8,25 @@ import (
 	"testing"
 	"time"
 
+	sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
 	"github.com/grafana/grafana/pkg/components/securejsondata"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 //nolint:goconst
 func TestDataSource_GetHttpTransport(t *testing.T) {
 	t.Run("Should use cached proxy", func(t *testing.T) {
+		var configuredTransport *http.Transport
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
+				configuredTransport = transport
+			},
+		})
+
 		clearDSProxyCache(t)
 		ds := DataSource{
 			Id:   1,
@@ -26,20 +34,30 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			Type: "Kubernetes",
 		}
 
-		tr1, err := ds.GetHttpTransport()
+		rt1, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt1)
+		tr1 := configuredTransport
 
-		tr2, err := ds.GetHttpTransport()
+		rt2, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt2)
+		tr2 := configuredTransport
 
 		require.Same(t, tr1, tr2)
 
-		assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
+		require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
-		assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
+		require.Empty(t, tr1.TLSClientConfig.Certificates)
-		assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
+		require.Nil(t, tr1.TLSClientConfig.RootCAs)
 	})
 
 	t.Run("Should not use cached proxy when datasource updated", func(t *testing.T) {
+		var configuredTransport *http.Transport
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
+				configuredTransport = transport
+			},
+		})
 		clearDSProxyCache(t)
 		setting.SecretKey = "password"
 
@@ -56,26 +74,36 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			Updated:        time.Now().Add(-2 * time.Minute),
 		}
 
-		tr1, err := ds.GetHttpTransport()
+		rt1, err := ds.GetHTTPTransport(provider)
+		require.NotNil(t, rt1)
 		require.NoError(t, err)
 
-		assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
+		tr1 := configuredTransport
-		assert.Empty(t, tr1.transport.TLSClientConfig.Certificates)
+
-		assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs)
+		require.False(t, tr1.TLSClientConfig.InsecureSkipVerify)
+		require.Empty(t, tr1.TLSClientConfig.Certificates)
+		require.Nil(t, tr1.TLSClientConfig.RootCAs)
 
 		ds.JsonData = nil
 		ds.SecureJsonData = map[string][]byte{}
 		ds.Updated = time.Now()
 
-		tr2, err := ds.GetHttpTransport()
+		rt2, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt2)
+		tr2 := configuredTransport
 
 		require.NotSame(t, tr1, tr2)
-
+		require.Nil(t, tr2.TLSClientConfig.RootCAs)
-		assert.Nil(t, tr2.transport.TLSClientConfig.RootCAs)
 	})
 
 	t.Run("Should set TLS client authentication enabled if configured in JsonData", func(t *testing.T) {
+		var configuredTransport *http.Transport
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
+				configuredTransport = transport
+			},
+		})
 		clearDSProxyCache(t)
 		setting.SecretKey = "password"
 
@@ -98,15 +126,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			},
 		}
 
-		tr, err := ds.GetHttpTransport()
+		rt, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt)
+		tr := configuredTransport
 
-		assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
+		require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
-		require.Len(t, tr.transport.TLSClientConfig.Certificates, 1)
+		require.Len(t, tr.TLSClientConfig.Certificates, 1)
 	})
 
 	t.Run("Should set user-supplied TLS CA if configured in JsonData", func(t *testing.T) {
+		var configuredTransport *http.Transport
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
+				configuredTransport = transport
+			},
+		})
 		clearDSProxyCache(t)
+		ClearDSDecryptionCache()
 		setting.SecretKey = "password"
 
 		json := simplejson.New()
@@ -126,15 +163,23 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			},
 		}
 
-		tr, err := ds.GetHttpTransport()
+		rt, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt)
+		tr := configuredTransport
 
-		assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify)
+		require.False(t, tr.TLSClientConfig.InsecureSkipVerify)
-		require.Len(t, tr.transport.TLSClientConfig.RootCAs.Subjects(), 1)
+		require.Len(t, tr.TLSClientConfig.RootCAs.Subjects(), 1)
-		assert.Equal(t, "server-name", tr.transport.TLSClientConfig.ServerName)
+		require.Equal(t, "server-name", tr.TLSClientConfig.ServerName)
 	})
 
 	t.Run("Should set skip TLS verification if configured in JsonData", func(t *testing.T) {
+		var configuredTransport *http.Transport
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
+				configuredTransport = transport
+			},
+		})
 		clearDSProxyCache(t)
 
 		json := simplejson.New()
@@ -147,19 +192,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			JsonData: json,
 		}
 
-		tr1, err := ds.GetHttpTransport()
+		rt1, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt1)
+		tr1 := configuredTransport
 
-		tr2, err := ds.GetHttpTransport()
+		rt2, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt2)
+		tr2 := configuredTransport
 
 		require.Same(t, tr1, tr2)
-
+		require.True(t, tr1.TLSClientConfig.InsecureSkipVerify)
-		assert.True(t, tr1.transport.TLSClientConfig.InsecureSkipVerify)
 	})
 
 	t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) {
+		provider := httpclient.NewProvider()
 		clearDSProxyCache(t)
+		ClearDSDecryptionCache()
 
 		json := simplejson.NewFromAny(map[string]interface{}{
 			"httpHeaderName1": "Authorization",
@@ -177,8 +227,8 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData},
 		}
 
-		headers := ds.getCustomHeaders()
+		headers := getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"})
-		assert.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
+		require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"])
 
 		// 1. Start HTTP test server which checks the request headers
 		backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -197,12 +247,13 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 
 		// 2. Get HTTP transport from datasource which uses the test server as backend
 		ds.Url = backend.URL
-		tr, err := ds.GetHttpTransport()
+		rt, err := ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
+		require.NotNil(t, rt)
 
 		// 3. Send test request which should have the Authorization header set
 		req := httptest.NewRequest("GET", backend.URL+"/test-headers", nil)
-		res, err := tr.RoundTrip(req)
+		res, err := rt.RoundTrip(req)
 		require.NoError(t, err)
 		t.Cleanup(func() {
 			err := res.Body.Close()
@@ -211,10 +262,11 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 		body, err := ioutil.ReadAll(res.Body)
 		require.NoError(t, err)
 		bodyStr := string(body)
-		assert.Equal(t, "Ok", bodyStr)
+		require.Equal(t, "Ok", bodyStr)
 	})
 
 	t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) {
+		provider := httpclient.NewProvider()
 		clearDSProxyCache(t)
 
 		json := simplejson.NewFromAny(map[string]interface{}{
@@ -227,47 +279,34 @@ func TestDataSource_GetHttpTransport(t *testing.T) {
 			JsonData: json,
 		}
 
-		client, err := ds.GetHttpClient()
+		client, err := ds.GetHTTPClient(provider)
 		require.NoError(t, err)
-
+		require.NotNil(t, client)
-		assert.Equal(t, 19*time.Second, client.Timeout)
+		require.Equal(t, 19*time.Second, client.Timeout)
 	})
 
-	t.Run("Should not include SigV4 middleware if not configured in JsonData", func(t *testing.T) {
+	t.Run("Should populate SigV4 options if configured in JsonData", func(t *testing.T) {
-		clearDSProxyCache(t)
+		var configuredOpts sdkhttpclient.Options
-
+		provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{
-		origEnabled := setting.SigV4AuthEnabled
+			ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) {
-		setting.SigV4AuthEnabled = true
+				configuredOpts = opts
-		t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
+			},
-
+		})
-		ds := DataSource{}
-
-		tr, err := ds.GetHttpTransport()
-		require.NoError(t, err)
-
-		_, ok := tr.next.(*http.Transport)
-		require.True(t, ok)
-	})
-
-	t.Run("Should not include SigV4 middleware if not configured in app config", func(t *testing.T) {
 		clearDSProxyCache(t)
 
-		origEnabled := setting.SigV4AuthEnabled
-		setting.SigV4AuthEnabled = false
-		t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled })
-
 		json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`))
 		require.NoError(t, err)
 
 		ds := DataSource{
+			Type:     DS_ES,
 			JsonData: json,
 		}
 
-		tr, err := ds.GetHttpTransport()
+		_, err = ds.GetHTTPTransport(provider)
 		require.NoError(t, err)
-
+		require.NotNil(t, configuredOpts)
-		_, ok := tr.next.(*http.Transport)
+		require.NotNil(t, configuredOpts.SigV4)
-		require.True(t, ok)
+		require.Equal(t, "es", configuredOpts.SigV4.Service)
 	})
 }
 
@@ -288,7 +327,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
 		// Populate cache
 		password, ok := ds.DecryptedValue("password")
 		require.True(t, ok)
-		assert.Equal(t, "password", password)
+		require.Equal(t, "password", password)
 
 		ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
 			"password": "",
@@ -296,7 +335,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
 
 		password, ok = ds.DecryptedValue("password")
 		require.True(t, ok)
-		assert.Equal(t, "password", password)
+		require.Equal(t, "password", password)
 	})
 
 	t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) {
@@ -315,7 +354,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
 		// Populate cache
 		password, ok := ds.DecryptedValue("password")
 		require.True(t, ok)
-		assert.Equal(t, "password", password)
+		require.Equal(t, "password", password)
 
 		ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{
 			"password": "",
@@ -324,7 +363,7 @@ func TestDataSource_DecryptedValue(t *testing.T) {
 
 		password, ok = ds.DecryptedValue("password")
 		require.True(t, ok)
-		assert.Empty(t, password)
+		require.Empty(t, password)
 	})
 }
 
@@ -334,7 +373,7 @@ func clearDSProxyCache(t *testing.T) {
 	ptc.Lock()
 	defer ptc.Unlock()
 
-	ptc.cache = make(map[int64]cachedTransport)
+	ptc.cache = make(map[int64]cachedRoundTripper)
 }
 
 const caCert string = `-----BEGIN CERTIFICATE-----

pkg/server/server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/bus"
 	_ "github.com/grafana/grafana/pkg/extensions"
+	"github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider"
 	"github.com/grafana/grafana/pkg/infra/localcache"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/metrics"
@@ -281,6 +282,7 @@ func (s *Server) buildServiceGraph(services []*registry.Descriptor) error {
 		s.cfg,
 		routing.NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(s.cfg)),
 		localcache.New(5*time.Minute, 10*time.Minute),
+		httpclientprovider.New(s.cfg),
 		s,
 	}
 	return registry.BuildServiceGraph(objs, services)

pkg/services/datasourceproxy/datasourceproxy.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/api/datasource"
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -25,6 +26,7 @@ type DatasourceProxyService struct {
 	PluginRequestValidator models.PluginRequestValidator `inject:""`
 	PluginManager          plugins.Manager               `inject:""`
 	Cfg                    *setting.Cfg                  `inject:""`
+	HTTPClientProvider     httpclient.Provider           `inject:""`
 }
 
 func (p *DatasourceProxyService) Init() error {
@@ -62,7 +64,7 @@ func (p *DatasourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte
 	}
 
 	proxyPath := getProxyPath(c)
-	proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg)
+	proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg, p.HTTPClientProvider)
 	if err != nil {
 		if errors.Is(err, datasource.URLValidationError{}) {
 			c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err)

pkg/setting/setting.go

@@ -81,6 +81,7 @@ var (
 	DataProxyTLSHandshakeTimeout   int
 	DataProxyExpectContinueTimeout int
 	DataProxyMaxIdleConns          int
+	DataProxyMaxIdleConnsPerHost   int
 	DataProxyKeepAlive             int
 	DataProxyIdleConnTimeout       int
 	StaticRootPath                 string
@@ -827,6 +828,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10)
 	DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1)
 	DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100)
+	DataProxyMaxIdleConnsPerHost = dataproxy.Key("max_idle_connections_per_host").MustInt(2)
 	DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)
 	cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false)
 

pkg/tsdb/azuremonitor/applicationinsights-datasource.go

@@ -241,8 +241,6 @@ func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInf
 		return nil, errutil.Wrap("Failed to create request", err)
 	}
 
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
-
 	pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg)
 
 	return req, nil

pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go

@@ -221,7 +221,6 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
 	}
 
 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
 
 	// find plugin
 	plugin := e.pluginManager.GetDataSource(dsInfo.Type)

pkg/tsdb/azuremonitor/azuremonitor-datasource.go

@@ -254,7 +254,6 @@ func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *mode
 	}
 
 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
 
 	pluginproxy.ApplyRoute(ctx, req, proxyPass, azureMonitorRoute, dsInfo, e.cfg)
 

pkg/tsdb/azuremonitor/azuremonitor.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"regexp"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -29,8 +30,9 @@ func init() {
 }
 
 type Service struct {
-	PluginManager plugins.Manager `inject:""`
+	PluginManager      plugins.Manager     `inject:""`
-	Cfg           *setting.Cfg    `inject:""`
+	HTTPClientProvider httpclient.Provider `inject:""`
+	Cfg                *setting.Cfg        `inject:""`
 }
 
 func (s *Service) Init() error {
@@ -48,7 +50,7 @@ type AzureMonitorExecutor struct {
 // NewAzureMonitorExecutor initializes a http client
 //nolint: staticcheck // plugins.DataPlugin deprecated
 func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-	httpClient, err := dsInfo.GetHttpClient()
+	httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider)
 	if err != nil {
 		return nil, err
 	}

pkg/tsdb/azuremonitor/insights-analytics-datasource.go

@@ -215,8 +215,6 @@ func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo
 		return nil, errutil.Wrap("Failed to create request", err)
 	}
 
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
-
 	pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg)
 
 	return req, nil

pkg/tsdb/cloudmonitoring/cloudmonitoring.go

@@ -22,6 +22,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/api/pluginproxy"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
@@ -75,8 +76,9 @@ func init() {
 }
 
 type Service struct {
-	PluginManager plugins.Manager `inject:""`
+	PluginManager      plugins.Manager     `inject:""`
-	Cfg           *setting.Cfg    `inject:""`
+	HTTPClientProvider httpclient.Provider `inject:""`
+	Cfg                *setting.Cfg        `inject:""`
 }
 
 func (s *Service) Init() error {
@@ -94,7 +96,7 @@ type Executor struct {
 // NewExecutor returns an Executor.
 //nolint: staticcheck // plugins.DataPlugin deprecated
 func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-	httpClient, err := dsInfo.GetHttpClient()
+	httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider)
 	if err != nil {
 		return nil, err
 	}
@@ -540,7 +542,6 @@ func (e *Executor) createRequest(ctx context.Context, dsInfo *models.DataSource,
 	}
 
 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
 
 	// find plugin
 	plugin := e.pluginManager.GetDataSource(dsInfo.Type)

pkg/tsdb/elasticsearch/client/client.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/tsdb/interval"
 
@@ -29,8 +30,8 @@ var (
 	clientLog = log.New(loggerName)
 )
 
-var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
+var newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) {
-	return ds.GetHttpClient()
+	return ds.GetHTTPClient(httpClientProvider)
 }
 
 // Client represents a client which can interact with elasticsearch api
@@ -72,7 +73,7 @@ func coerceVersion(v *simplejson.Json) (*semver.Version, error) {
 }
 
 // NewClient creates a new elasticsearch client
-var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) {
+var NewClient = func(ctx context.Context, httpClientProvider httpclient.Provider, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) {
 	version, err := coerceVersion(ds.JsonData.Get("esVersion"))
 
 	if err != nil {
@@ -108,13 +109,14 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugi
 }
 
 type baseClientImpl struct {
-	ctx          context.Context
+	ctx                context.Context
-	ds           *models.DataSource
+	httpClientProvider httpclient.Provider
-	version      *semver.Version
+	ds                 *models.DataSource
-	timeField    string
+	version            *semver.Version
-	indices      []string
+	timeField          string
-	timeRange    plugins.DataTimeRange
+	indices            []string
-	debugEnabled bool
+	timeRange          plugins.DataTimeRange
+	debugEnabled       bool
 }
 
 func (c *baseClientImpl) GetVersion() *semver.Version {
@@ -208,20 +210,9 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 		}
 	}
 
-	req.Header.Set("User-Agent", "Grafana")
 	req.Header.Set("Content-Type", "application/x-ndjson")
 
-	if c.ds.BasicAuth {
+	httpClient, err := newDatasourceHttpClient(c.httpClientProvider, c.ds)
-		clientLog.Debug("Request configured to use basic authentication")
-		req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.DecryptedBasicAuthPassword())
-	}
-
-	if !c.ds.BasicAuth && c.ds.User != "" {
-		clientLog.Debug("Request configured to use basic authentication")
-		req.SetBasicAuth(c.ds.User, c.ds.DecryptedPassword())
-	}
-
-	httpClient, err := newDatasourceHttpClient(c.ds)
 	if err != nil {
 		return nil, err
 	}

pkg/tsdb/elasticsearch/client/client_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/tsdb/interval"
@@ -24,7 +25,7 @@ func TestNewClient(t *testing.T) {
 			JsonData: simplejson.NewFromAny(make(map[string]interface{})),
 		}
 
-		_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+		_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 		require.Error(t, err)
 	})
 
@@ -35,7 +36,7 @@ func TestNewClient(t *testing.T) {
 			}),
 		}
 
-		_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+		_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 		require.Error(t, err)
 	})
 
@@ -48,7 +49,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.Error(t, err)
 		})
 
@@ -60,7 +61,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.NoError(t, err)
 			assert.Equal(t, "2.0.0", c.GetVersion().String())
 		})
@@ -73,7 +74,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.NoError(t, err)
 			assert.Equal(t, "5.0.0", c.GetVersion().String())
 		})
@@ -86,7 +87,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.NoError(t, err)
 			assert.Equal(t, "5.6.0", c.GetVersion().String())
 		})
@@ -99,7 +100,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.NoError(t, err)
 			assert.Equal(t, "6.0.0", c.GetVersion().String())
 		})
@@ -112,7 +113,7 @@ func TestNewClient(t *testing.T) {
 				}),
 			}
 
-			c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+			c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 			require.NoError(t, err)
 			assert.Equal(t, "7.0.0", c.GetVersion().String())
 		})
@@ -127,7 +128,7 @@ func TestNewClient(t *testing.T) {
 			}),
 		}
 
-		c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+		c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 		require.NoError(t, err)
 		assert.Equal(t, version, c.GetVersion().String())
 	})
@@ -141,7 +142,7 @@ func TestNewClient(t *testing.T) {
 			}),
 		}
 
-		_, err := NewClient(context.Background(), ds, plugins.DataTimeRange{})
+		_, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{})
 		require.Error(t, err)
 	})
 }
@@ -408,14 +409,14 @@ func httpClientScenario(t *testing.T, desc string, ds *models.DataSource, fn sce
 		toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
 		timeRange := plugins.NewDataTimeRange(fromStr, toStr)
 
-		c, err := NewClient(context.Background(), ds, timeRange)
+		c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, timeRange)
 		require.NoError(t, err)
 		require.NotNil(t, c)
 		sc.client = c
 
 		currentNewDatasourceHTTPClient := newDatasourceHttpClient
 
-		newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
+		newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) {
 			return ts.Client(), nil
 		}
 

pkg/tsdb/elasticsearch/elasticsearch.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client"
@@ -12,15 +13,20 @@ import (
 
 // ElasticsearchExecutor represents a handler for handling elasticsearch datasource request
 type Executor struct {
+	httpClientProvider httpclient.Provider
 	intervalCalculator interval.Calculator
 }
 
-// NewExecutor creates a new Executor.
+// New creates a new Executor func.
-//nolint: staticcheck // plugins.DataPlugin deprecated
+// nolint:staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	return &Executor{
+	// nolint:staticcheck // plugins.DataPlugin deprecated
-		intervalCalculator: interval.NewCalculator(),
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-	}, nil
+		return &Executor{
+			httpClientProvider: httpClientProvider,
+			intervalCalculator: interval.NewCalculator(),
+		}, nil
+	}
 }
 
 // Query handles an elasticsearch datasource request
@@ -31,8 +37,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
 		return plugins.DataResponse{}, fmt.Errorf("query contains no queries")
 	}
 
-	client, err := es.NewClient(ctx, dsInfo, *tsdbQuery.TimeRange)
+	client, err := es.NewClient(ctx, e.httpClientProvider, dsInfo, *tsdbQuery.TimeRange)
-
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}

pkg/tsdb/graphite/graphite.go

@@ -15,6 +15,7 @@ import (
 
 	"golang.org/x/net/context/ctxhttp"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -23,12 +24,17 @@ import (
 )
 
 type GraphiteExecutor struct {
-	HttpClient *http.Client
+	httpClientProvider httpclient.Provider
 }
 
-//nolint: staticcheck // plugins.DataPlugin deprecated
+// nolint:staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	return &GraphiteExecutor{}, nil
+	// nolint:staticcheck // plugins.DataPlugin deprecated
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
+		return &GraphiteExecutor{
+			httpClientProvider: httpClientProvider,
+		}, nil
+	}
 }
 
 var glog = log.New("tsdb.graphite")
@@ -91,7 +97,7 @@ func (e *GraphiteExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou
 		return plugins.DataResponse{}, err
 	}
 
-	httpClient, err := dsInfo.GetHttpClient()
+	httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}

pkg/tsdb/influxdb/flux/executor_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
 	"github.com/grafana/grafana/pkg/components/securejsondata"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -226,7 +227,7 @@ func TestRealQuery(t *testing.T) {
 			}),
 		}
 
-		runner, err := runnerFromDataSource(dsInfo)
+		runner, err := runnerFromDataSource(httpclient.NewProvider(), dsInfo)
 		require.NoError(t, err)
 
 		dr := executeQuery(context.Background(), queryModel{

pkg/tsdb/influxdb/flux/flux.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -22,13 +23,13 @@ func init() {
 
 // Query builds flux queries, executes them, and returns the results.
 //nolint: staticcheck // plugins.DataQuery deprecated
-func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
+func Query(ctx context.Context, httpClientProvider httpclient.Provider, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) (
 	plugins.DataResponse, error) {
 	glog.Debug("Received a query", "query", tsdbQuery)
 	tRes := plugins.DataResponse{
 		Results: make(map[string]plugins.DataQueryResult),
 	}
-	r, err := runnerFromDataSource(dsInfo)
+	r, err := runnerFromDataSource(httpClientProvider, dsInfo)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}
@@ -69,7 +70,7 @@ func (r *runner) runQuery(ctx context.Context, fluxQuery string) (*api.QueryTabl
 }
 
 // runnerFromDataSource creates a runner from the datasource model (the datasource instance's configuration).
-func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) {
+func runnerFromDataSource(httpClientProvider httpclient.Provider, dsInfo *models.DataSource) (*runner, error) {
 	org := dsInfo.JsonData.Get("organization").MustString("")
 	if org == "" {
 		return nil, fmt.Errorf("missing organization in datasource configuration")
@@ -85,7 +86,7 @@ func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) {
 	}
 
 	opts := influxdb2.DefaultOptions()
-	hc, err := dsInfo.GetHttpClient()
+	hc, err := dsInfo.GetHTTPClient(httpClientProvider)
 	if err != nil {
 		return nil, err
 	}

pkg/tsdb/influxdb/influxdb.go

@@ -9,6 +9,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -17,17 +18,21 @@ import (
 )
 
 type Executor struct {
-	// *models.DataSource
+	httpClientProvider httpclient.Provider
-	QueryParser    *InfluxdbQueryParser
+	QueryParser        *InfluxdbQueryParser
-	ResponseParser *ResponseParser
+	ResponseParser     *ResponseParser
 }
 
-//nolint: staticcheck // plugins.DataPlugin deprecated
+// nolint:staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	return &Executor{
+	// nolint:staticcheck // plugins.DataPlugin deprecated
-		QueryParser:    &InfluxdbQueryParser{},
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-		ResponseParser: &ResponseParser{},
+		return &Executor{
-	}, nil
+			httpClientProvider: httpClientProvider,
+			QueryParser:        &InfluxdbQueryParser{},
+			ResponseParser:     &ResponseParser{},
+		}, nil
+	}
 }
 
 var (
@@ -47,7 +52,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd
 
 	version := dsInfo.JsonData.Get("version").MustString("")
 	if version == "Flux" {
-		return flux.Query(ctx, dsInfo, tsdbQuery)
+		return flux.Query(ctx, e.httpClientProvider, dsInfo, tsdbQuery)
 	}
 
 	glog.Debug("Making a non-Flux type query")
@@ -74,7 +79,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd
 		return plugins.DataResponse{}, err
 	}
 
-	httpClient, err := dsInfo.GetHttpClient()
+	httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}

pkg/tsdb/loki/loki.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -23,16 +24,17 @@ import (
 
 type LokiExecutor struct {
 	intervalCalculator interval.Calculator
+	httpClientProvider httpclient.Provider
 }
 
-//nolint: staticcheck // plugins.DataPlugin deprecated
+// nolint:staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-	return newExecutor(), nil
+	// nolint:staticcheck // plugins.DataPlugin deprecated
-}
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-
+		return &LokiExecutor{
-func newExecutor() *LokiExecutor {
+			intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
-	return &LokiExecutor{
+			httpClientProvider: httpClientProvider,
-		intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
+		}, nil
 	}
 }
 
@@ -49,12 +51,12 @@ func (e *LokiExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource,
 		Results: map[string]plugins.DataQueryResult{},
 	}
 
-	tlsConfig, err := dsInfo.GetTLSConfig()
+	tlsConfig, err := dsInfo.GetTLSConfig(e.httpClientProvider)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}
 
-	transport, err := dsInfo.GetHttpTransport()
+	transport, err := dsInfo.GetHTTPTransport(e.httpClientProvider)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}

pkg/tsdb/loki/loki_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/loki/pkg/loghttp"
@@ -60,9 +61,10 @@ func TestLoki(t *testing.T) {
 			TimeRange: &timeRange,
 		}
 
-		exe := newExecutor()
+		exe, err := New(httpclient.NewProvider())(dsInfo)
 		require.NoError(t, err)
-		models, err := exe.parseQuery(dsInfo, queryContext)
+		lokiExecutor := exe.(*LokiExecutor)
+		models, err := lokiExecutor.parseQuery(dsInfo, queryContext)
 		require.NoError(t, err)
 		require.Equal(t, time.Second*30, models[0].Step)
 	})
@@ -82,15 +84,16 @@ func TestLoki(t *testing.T) {
 				{Model: jsonModel},
 			},
 		}
-		exe := newExecutor()
+		exe, err := New(httpclient.NewProvider())(dsInfo)
 		require.NoError(t, err)
-		models, err := exe.parseQuery(dsInfo, queryContext)
+		lokiExecutor := exe.(*LokiExecutor)
+		models, err := lokiExecutor.parseQuery(dsInfo, queryContext)
 		require.NoError(t, err)
 		require.Equal(t, time.Minute*2, models[0].Step)
 
 		timeRange = plugins.NewDataTimeRange("1h", "now")
 		queryContext.TimeRange = &timeRange
-		models, err = exe.parseQuery(dsInfo, queryContext)
+		models, err = lokiExecutor.parseQuery(dsInfo, queryContext)
 		require.NoError(t, err)
 		require.Equal(t, time.Second*2, models[0].Step)
 	})

pkg/tsdb/mysql/mysql.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VividCortex/mysqlerr"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/go-sql-driver/mysql"
@@ -32,52 +33,55 @@ func characterEscape(s string, escapeChar string) string {
 }
 
 //nolint: staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSource) (plugins.DataPlugin, error) {
-	logger := log.New("tsdb.mysql")
+	//nolint: staticcheck // plugins.DataPlugin deprecated
+	return func(datasource *models.DataSource) (plugins.DataPlugin, error) {
+		logger := log.New("tsdb.mysql")
 
-	protocol := "tcp"
+		protocol := "tcp"
-	if strings.HasPrefix(datasource.Url, "/") {
+		if strings.HasPrefix(datasource.Url, "/") {
-		protocol = "unix"
+			protocol = "unix"
-	}
+		}
-
-	cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
-		characterEscape(datasource.User, ":"),
-		datasource.DecryptedPassword(),
-		protocol,
-		characterEscape(datasource.Url, ")"),
-		characterEscape(datasource.Database, "?"),
-	)
 
-	tlsConfig, err := datasource.GetTLSConfig()
+		cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true",
-	if err != nil {
+			characterEscape(datasource.User, ":"),
-		return nil, err
+			datasource.DecryptedPassword(),
-	}
+			protocol,
+			characterEscape(datasource.Url, ")"),
+			characterEscape(datasource.Database, "?"),
+		)
 
-	if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
+		tlsConfig, err := datasource.GetTLSConfig(httpClientProvider)
-		tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
+		if err != nil {
-		if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
 			return nil, err
 		}
-		cnnstr += "&tls=" + tlsConfigString
-	}
 
-	if setting.Env == setting.Dev {
+		if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 {
-		logger.Debug("getEngine", "connection", cnnstr)
+			tlsConfigString := fmt.Sprintf("ds%d", datasource.Id)
-	}
+			if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil {
+				return nil, err
+			}
+			cnnstr += "&tls=" + tlsConfigString
+		}
 
-	config := sqleng.DataPluginConfiguration{
+		if setting.Env == setting.Dev {
-		DriverName:        "mysql",
+			logger.Debug("getEngine", "connection", cnnstr)
-		ConnectionString:  cnnstr,
+		}
-		Datasource:        datasource,
-		TimeColumnNames:   []string{"time", "time_sec"},
-		MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
-	}
 
-	rowTransformer := mysqlQueryResultTransformer{
+		config := sqleng.DataPluginConfiguration{
-		log: logger,
+			DriverName:        "mysql",
-	}
+			ConnectionString:  cnnstr,
+			Datasource:        datasource,
+			TimeColumnNames:   []string{"time", "time_sec"},
+			MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"},
+		}
 
-	return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
+		rowTransformer := mysqlQueryResultTransformer{
+			log: logger,
+		}
+
+		return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger)
+	}
 }
 
 type mysqlQueryResultTransformer struct {

pkg/tsdb/mysql/mysql_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/components/securejsondata"
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
@@ -56,7 +57,7 @@ func TestMySQL(t *testing.T) {
 		return sql, nil
 	}
 
-	exe, err := NewExecutor(&models.DataSource{
+	exe, err := New(httpclient.NewProvider())(&models.DataSource{
 		JsonData:       simplejson.New(),
 		SecureJsonData: securejsondata.SecureJsonData{},
 	})

pkg/tsdb/opentsdb/opentsdb.go

@@ -15,6 +15,7 @@ import (
 	"net/url"
 
 	"github.com/grafana/grafana/pkg/components/null"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -22,11 +23,17 @@ import (
 )
 
 type OpenTsdbExecutor struct {
+	httpClientProvider httpclient.Provider
 }
 
-// nolint:staticcheck // plugins.DataQueryResult deprecated
+//nolint: staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	return &OpenTsdbExecutor{}, nil
+	//nolint: staticcheck // plugins.DataPlugin deprecated
+	return func(*models.DataSource) (plugins.DataPlugin, error) {
+		return &OpenTsdbExecutor{
+			httpClientProvider: httpClientProvider,
+		}, nil
+	}
 }
 
 var (
@@ -56,7 +63,7 @@ func (e *OpenTsdbExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou
 		return plugins.DataResponse{}, err
 	}
 
-	httpClient, err := dsInfo.GetHttpClient()
+	httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider)
 	if err != nil {
 		return plugins.DataResponse{}, err
 	}

pkg/tsdb/prometheus/prometheus.go

@@ -14,6 +14,7 @@ import (
 	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -65,13 +66,20 @@ func (transport *prometheusTransport) RoundTrip(req *http.Request) (*http.Respon
 }
 
 //nolint: staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
+func New(provider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	return &PrometheusExecutor{
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-		intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
+		transport, err := dsInfo.GetHTTPTransport(provider)
-		baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) {
+		if err != nil {
-			return ds.GetHttpTransport()
+			return nil, err
-		},
+		}
-	}, nil
+
+		return &PrometheusExecutor{
+			intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}),
+			baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) {
+				return transport, nil
+			},
+		}, nil
+	}
 }
 
 var (

pkg/tsdb/prometheus/prometheus_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	p "github.com/prometheus/common/model"
@@ -21,7 +22,7 @@ func TestPrometheus(t *testing.T) {
 	dsInfo := &models.DataSource{
 		JsonData: json,
 	}
-	plug, err := NewExecutor(dsInfo)
+	plug, err := New(httpclient.NewProvider())(dsInfo)
 	require.NoError(t, err)
 	executor := plug.(*PrometheusExecutor)
 

pkg/tsdb/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/registry"
@@ -47,6 +48,7 @@ type Service struct {
 	CloudMonitoringService *cloudmonitoring.Service      `inject:""`
 	AzureMonitorService    *azuremonitor.Service         `inject:""`
 	PluginManager          plugins.Manager               `inject:""`
+	HTTPClientProvider     httpclient.Provider           `inject:""`
 
 	//nolint: staticcheck // plugins.DataPlugin deprecated
 	registry map[string]func(*models.DataSource) (plugins.DataPlugin, error)
@@ -54,18 +56,18 @@ type Service struct {
 
 // Init initialises the service.
 func (s *Service) Init() error {
-	s.registry["graphite"] = graphite.NewExecutor
+	s.registry["graphite"] = graphite.New(s.HTTPClientProvider)
-	s.registry["opentsdb"] = opentsdb.NewExecutor
+	s.registry["opentsdb"] = opentsdb.New(s.HTTPClientProvider)
-	s.registry["prometheus"] = prometheus.NewExecutor
+	s.registry["prometheus"] = prometheus.New(s.HTTPClientProvider)
-	s.registry["influxdb"] = influxdb.NewExecutor
+	s.registry["influxdb"] = influxdb.New(s.HTTPClientProvider)
 	s.registry["mssql"] = mssql.NewExecutor
 	s.registry["postgres"] = s.PostgresService.NewExecutor
-	s.registry["mysql"] = mysql.NewExecutor
+	s.registry["mysql"] = mysql.New(s.HTTPClientProvider)
-	s.registry["elasticsearch"] = elasticsearch.NewExecutor
+	s.registry["elasticsearch"] = elasticsearch.New(s.HTTPClientProvider)
 	s.registry["stackdriver"] = s.CloudMonitoringService.NewExecutor
 	s.registry["grafana-azure-monitor-datasource"] = s.AzureMonitorService.NewExecutor
-	s.registry["loki"] = loki.NewExecutor
+	s.registry["loki"] = loki.New(s.HTTPClientProvider)
-	s.registry["tempo"] = tempo.NewExecutor
+	s.registry["tempo"] = tempo.New(s.HTTPClientProvider)
 	return nil
 }
 

pkg/tsdb/tempo/tempo.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -18,17 +19,20 @@ type tempoExecutor struct {
 	httpClient *http.Client
 }
 
-// NewExecutor returns a tempoExecutor.DataQueryResult
+// NewExecutor returns a tempoExecutor.
 //nolint: staticcheck // plugins.DataPlugin deprecated
-func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
+func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) {
-	httpClient, err := dsInfo.GetHttpClient()
+	//nolint: staticcheck // plugins.DataPlugin deprecated
-	if err != nil {
+	return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) {
-		return nil, err
+		httpClient, err := dsInfo.GetHTTPClient(httpClientProvider)
-	}
+		if err != nil {
+			return nil, err
+		}
 
-	return &tempoExecutor{
+		return &tempoExecutor{
-		httpClient: httpClient,
+			httpClient: httpClient,
-	}, nil
+		}, nil
+	}
 }
 
 var (

pkg/tsdb/tempo/tempo_test.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestTempo(t *testing.T) {
-	plug, err := NewExecutor(&models.DataSource{})
+	plug, err := New(httpclient.NewProvider())(&models.DataSource{})
 	executor := plug.(*tempoExecutor)
 	require.NoError(t, err)