From 348e76fc8e176d61adf3ea1edddf4b40e9fbfbe4 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Wed, 19 May 2021 23:53:41 +0200 Subject: [PATCH] Datasource: Shared HTTP client provider for core backend data sources and any data source using the data source proxy (#33439) Uses new httpclient package from grafana-plugin-sdk-go introduced via grafana/grafana-plugin-sdk-go#328. Replaces the GetHTTPClient, GetTransport, GetTLSConfig methods defined on DataSource model. Longer-term the goal is to migrate core HTTP backend data sources to use the SDK contracts and using httpclient.Provider for creating HTTP clients and such. Co-authored-by: Arve Knudsen --- conf/defaults.ini | 3 + conf/sample.ini | 3 + devenv/docker/blocks/slow_proxy/.env | 2 + .../blocks/slow_proxy/docker-compose.yaml | 3 +- devenv/docker/blocks/slow_proxy/main.go | 13 +- docs/sources/administration/configuration.md | 4 + pkg/api/pluginproxy/ds_proxy.go | 33 ++- pkg/api/pluginproxy/ds_proxy_test.go | 76 +++-- .../datasource_metrics_middleware.go | 105 +++++++ .../datasource_metrics_middleware_test.go | 130 +++++++++ .../http_client_provider.go | 30 ++ .../http_client_provider_test.go | 52 ++++ .../httpclientprovider/sigv4_middleware.go | 36 +++ .../sigv4_middleware_test.go | 89 ++++++ .../httpclient/httpclientprovider/testing.go | 18 ++ .../user_agent_middleware.go | 27 ++ .../user_agent_middleware_test.go | 82 ++++++ pkg/infra/httpclient/provider.go | 32 +++ pkg/models/datasource_cache.go | 270 ++++++------------ pkg/models/datasource_cache_test.go | 163 +++++++---- pkg/server/server.go | 2 + .../datasourceproxy/datasourceproxy.go | 4 +- pkg/setting/setting.go | 2 + .../applicationinsights-datasource.go | 2 - .../azure-log-analytics-datasource.go | 1 - .../azuremonitor/azuremonitor-datasource.go | 1 - pkg/tsdb/azuremonitor/azuremonitor.go | 8 +- .../insights-analytics-datasource.go | 2 - pkg/tsdb/cloudmonitoring/cloudmonitoring.go | 9 +- pkg/tsdb/elasticsearch/client/client.go | 35 +-- pkg/tsdb/elasticsearch/client/client_test.go | 25 +- pkg/tsdb/elasticsearch/elasticsearch.go | 21 +- pkg/tsdb/graphite/graphite.go | 16 +- pkg/tsdb/influxdb/flux/executor_test.go | 3 +- pkg/tsdb/influxdb/flux/flux.go | 9 +- pkg/tsdb/influxdb/influxdb.go | 27 +- pkg/tsdb/loki/loki.go | 22 +- pkg/tsdb/loki/loki_test.go | 13 +- pkg/tsdb/mysql/mysql.go | 78 ++--- pkg/tsdb/mysql/mysql_test.go | 3 +- pkg/tsdb/opentsdb/opentsdb.go | 15 +- pkg/tsdb/prometheus/prometheus.go | 22 +- pkg/tsdb/prometheus/prometheus_test.go | 3 +- pkg/tsdb/service.go | 18 +- pkg/tsdb/tempo/tempo.go | 22 +- pkg/tsdb/tempo/tempo_test.go | 3 +- 46 files changed, 1076 insertions(+), 461 deletions(-) create mode 100644 devenv/docker/blocks/slow_proxy/.env create mode 100644 pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go create mode 100644 pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go create mode 100644 pkg/infra/httpclient/httpclientprovider/http_client_provider.go create mode 100644 pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go create mode 100644 pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go create mode 100644 pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go create mode 100644 pkg/infra/httpclient/httpclientprovider/testing.go create mode 100644 pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go create mode 100644 pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go create mode 100644 pkg/infra/httpclient/provider.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 29b3d693515..8b8511aec78 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -158,6 +158,9 @@ expect_continue_timeout_seconds = 1 # The maximum number of idle connections that Grafana will keep alive. max_idle_connections = 100 +# The maximum number of idle connections per host that Grafana will keep alive. +max_idle_connections_per_host = 2 + # How many seconds the data proxy keeps an idle connection open before timing out. idle_conn_timeout_seconds = 90 diff --git a/conf/sample.ini b/conf/sample.ini index 2dda4f1aa0c..18e792a39b9 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -164,6 +164,9 @@ # The maximum number of idle connections that Grafana will keep alive. ;max_idle_connections = 100 +# The maximum number of idle connections per host that Grafana will keep alive. +;max_idle_connections_per_host = 2 + # How many seconds the data proxy keeps an idle connection open before timing out. ;idle_conn_timeout_seconds = 90 diff --git a/devenv/docker/blocks/slow_proxy/.env b/devenv/docker/blocks/slow_proxy/.env new file mode 100644 index 00000000000..bd169d2d680 --- /dev/null +++ b/devenv/docker/blocks/slow_proxy/.env @@ -0,0 +1,2 @@ +ORIGIN_SERVER=http://localhost:9090/ +SLEEP_DURATION=60s \ No newline at end of file diff --git a/devenv/docker/blocks/slow_proxy/docker-compose.yaml b/devenv/docker/blocks/slow_proxy/docker-compose.yaml index 4af9396a510..a74c6539644 100644 --- a/devenv/docker/blocks/slow_proxy/docker-compose.yaml +++ b/devenv/docker/blocks/slow_proxy/docker-compose.yaml @@ -4,4 +4,5 @@ ports: - "3011:3011" environment: - ORIGIN_SERVER: "http://localhost:9090/" \ No newline at end of file + ORIGIN_SERVER: ${ORIGIN_SERVER} + SLEEP_DURATION: ${SLEEP_DURATION} \ No newline at end of file diff --git a/devenv/docker/blocks/slow_proxy/main.go b/devenv/docker/blocks/slow_proxy/main.go index 0db80084913..d571da0ee98 100644 --- a/devenv/docker/blocks/slow_proxy/main.go +++ b/devenv/docker/blocks/slow_proxy/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "log" "net/http" "net/http/httputil" @@ -16,13 +15,21 @@ func main() { origin = "http://localhost:9090/" } - sleep := time.Minute + sleepDurationStr := os.Getenv("SLEEP_DURATION") + if sleepDurationStr == "" { + sleepDurationStr = "60s" + } + + sleep, err := time.ParseDuration(sleepDurationStr) + if err != nil { + log.Fatalf("failed to parse SLEEP_DURATION: %v", err) + } originURL, _ := url.Parse(origin) proxy := httputil.NewSingleHostReverseProxy(originURL) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - fmt.Printf("sleeping for %s then proxying request: %s", sleep.String(), r.RequestURI) + log.Printf("sleeping for %s then proxying request: url '%s', headers: '%v'", sleep.String(), r.RequestURI, r.Header) <-time.After(sleep) proxy.ServeHTTP(w, r) }) diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md index 441dcba95da..f1db5410468 100644 --- a/docs/sources/administration/configuration.md +++ b/docs/sources/administration/configuration.md @@ -418,6 +418,10 @@ The length of time that Grafana will wait for a datasource’s first response he The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation. +### max_idle_connections_per_host + +The maximum number of idle connections per host that Grafana will maintain. Default is `2`. For more details check the [Transport.MaxIdleConnsPerHost](https://golang.org/pkg/net/http/#Transport.MaxIdleConnsPerHost) documentation. + ### idle_conn_timeout_seconds The length of time that Grafana maintains idle connections before closing them. Default is `90` seconds. For more details check the [Transport.IdleConnTimeout](https://golang.org/pkg/net/http/#Transport.IdleConnTimeout) documentation. diff --git a/pkg/api/pluginproxy/ds_proxy.go b/pkg/api/pluginproxy/ds_proxy.go index 23bdcba17ba..2e20e6a3c1b 100644 --- a/pkg/api/pluginproxy/ds_proxy.go +++ b/pkg/api/pluginproxy/ds_proxy.go @@ -14,6 +14,7 @@ import ( "time" "github.com/grafana/grafana/pkg/api/datasource" + "github.com/grafana/grafana/pkg/infra/httpclient" glog "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -30,13 +31,14 @@ var ( ) type DataSourceProxy struct { - ds *models.DataSource - ctx *models.ReqContext - targetUrl *url.URL - proxyPath string - route *plugins.AppPluginRoute - plugin *plugins.DataSourcePlugin - cfg *setting.Cfg + ds *models.DataSource + ctx *models.ReqContext + targetUrl *url.URL + proxyPath string + route *plugins.AppPluginRoute + plugin *plugins.DataSourcePlugin + cfg *setting.Cfg + clientProvider httpclient.Provider } type handleResponseTransport struct { @@ -69,19 +71,20 @@ func (lw *logWrapper) Write(p []byte) (n int, err error) { // NewDataSourceProxy creates a new Datasource proxy func NewDataSourceProxy(ds *models.DataSource, plugin *plugins.DataSourcePlugin, ctx *models.ReqContext, - proxyPath string, cfg *setting.Cfg) (*DataSourceProxy, error) { + proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider) (*DataSourceProxy, error) { targetURL, err := datasource.ValidateURL(ds.Type, ds.Url) if err != nil { return nil, err } return &DataSourceProxy{ - ds: ds, - plugin: plugin, - ctx: ctx, - proxyPath: proxyPath, - targetUrl: targetURL, - cfg: cfg, + ds: ds, + plugin: plugin, + ctx: ctx, + proxyPath: proxyPath, + targetUrl: targetURL, + cfg: cfg, + clientProvider: clientProvider, }, nil } @@ -101,7 +104,7 @@ func (proxy *DataSourceProxy) HandleRequest() { proxyErrorLogger := logger.New("userId", proxy.ctx.UserId, "orgId", proxy.ctx.OrgId, "uname", proxy.ctx.Login, "path", proxy.ctx.Req.URL.Path, "remote_addr", proxy.ctx.RemoteAddr(), "referer", proxy.ctx.Req.Referer()) - transport, err := proxy.ds.GetHttpTransport() + transport, err := proxy.ds.GetHTTPTransport(proxy.clientProvider) if err != nil { proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err) return diff --git a/pkg/api/pluginproxy/ds_proxy_test.go b/pkg/api/pluginproxy/ds_proxy_test.go index 36e491094ef..a6650963f01 100644 --- a/pkg/api/pluginproxy/ds_proxy_test.go +++ b/pkg/api/pluginproxy/ds_proxy_test.go @@ -12,23 +12,24 @@ import ( "time" "github.com/grafana/grafana/pkg/api/datasource" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/securejsondata" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/login/social" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/oauth2" macaron "gopkg.in/macaron.v1" - - "github.com/grafana/grafana/pkg/bus" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/login/social" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/util" ) func TestDataSourceProxy_routeRule(t *testing.T) { + httpClientProvider := httpclient.NewProvider() + t.Run("Plugin with routes", func(t *testing.T) { plugin := &plugins.DataSourcePlugin{ Routes: []*plugins.AppPluginRoute{ @@ -113,7 +114,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path", func(t *testing.T) { ctx, req := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider) require.NoError(t, err) proxy.route = plugin.Routes[0] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg) @@ -124,7 +125,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic url", func(t *testing.T) { ctx, req := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/common/some/method", cfg, httpClientProvider) require.NoError(t, err) proxy.route = plugin.Routes[3] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg) @@ -135,7 +136,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path with no url", func(t *testing.T) { ctx, req := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpClientProvider) require.NoError(t, err) proxy.route = plugin.Routes[4] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg) @@ -145,7 +146,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("When matching route path and has dynamic body", func(t *testing.T) { ctx, req := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/body", cfg, httpClientProvider) require.NoError(t, err) proxy.route = plugin.Routes[5] ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, proxy.route, proxy.ds, cfg) @@ -158,7 +159,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("Validating request", func(t *testing.T) { t.Run("plugin route with valid role", func(t *testing.T) { ctx, _ := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/v4/some/method", cfg, httpClientProvider) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -166,7 +167,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is editor", func(t *testing.T) { ctx, _ := setUp() - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider) require.NoError(t, err) err = proxy.validateRequest() require.Error(t, err) @@ -175,7 +176,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("plugin route with admin role and user is admin", func(t *testing.T) { ctx, _ := setUp() ctx.SignedInUser.OrgRole = models.ROLE_ADMIN - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "api/admin", cfg, httpClientProvider) require.NoError(t, err) err = proxy.validateRequest() require.NoError(t, err) @@ -257,7 +258,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { cfg := &setting.Cfg{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg) @@ -272,7 +273,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { req, err := http.NewRequest("GET", "http://localhost/asd", nil) require.NoError(t, err) client = newFakeHTTPClient(t, json2) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken2", cfg, httpClientProvider) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[1], proxy.ds, cfg) @@ -288,7 +289,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { require.NoError(t, err) client = newFakeHTTPClient(t, []byte{}) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "pathwithtoken1", cfg, httpClientProvider) require.NoError(t, err) ApplyRoute(proxy.ctx.Req.Context(), req, proxy.proxyPath, plugin.Routes[0], proxy.ds, cfg) @@ -304,17 +305,11 @@ func TestDataSourceProxy_routeRule(t *testing.T) { }) t.Run("When proxying graphite", func(t *testing.T) { - origBuildVer := setting.BuildVersion - t.Cleanup(func() { - setting.BuildVersion = origBuildVer - }) - setting.BuildVersion = "5.3.0" - plugin := &plugins.DataSourcePlugin{} ds := &models.DataSource{Url: "htttp://graphite:8080", Type: models.DS_GRAPHITE} ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{BuildVersion: "5.3.0"}, httpClientProvider) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -324,7 +319,6 @@ func TestDataSourceProxy_routeRule(t *testing.T) { t.Run("Can translate request URL and path", func(t *testing.T) { assert.Equal(t, "graphite:8080", req.URL.Host) assert.Equal(t, "/render", req.URL.Path) - assert.Equal(t, "Grafana/5.3.0", req.Header.Get("User-Agent")) }) }) @@ -340,7 +334,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -363,7 +357,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -390,7 +384,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { } ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) requestURL, err := url.Parse("http://grafana.com/sub") @@ -411,7 +405,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { Url: "http://host/root/", } ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) req.Header.Set("Origin", "grafana.com") @@ -472,7 +466,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { Req: macaron.Request{Request: req}, }, } - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/to/folder/", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) req, err = http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -545,6 +539,7 @@ func TestDataSourceProxy_routeRule(t *testing.T) { // test DataSourceProxy request handling. func TestDataSourceProxy_requestHandling(t *testing.T) { + httpClientProvider := httpclient.NewProvider() var writeErr error plugin := &plugins.DataSourcePlugin{} @@ -605,7 +600,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Run("When response header Set-Cookie is not set should remove proxied Set-Cookie header", func(t *testing.T) { ctx, ds := setUp(t) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) proxy.HandleRequest() @@ -620,7 +615,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { "Set-Cookie": "important_cookie=important_value", }, }) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) proxy.HandleRequest() @@ -639,7 +634,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { t.Log("Wrote 401 response") }, }) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/render", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) proxy.HandleRequest() @@ -661,7 +656,7 @@ func TestDataSourceProxy_requestHandling(t *testing.T) { }) ctx.Req.Request = httptest.NewRequest("GET", "/api/datasources/proxy/1/path/%2Ftest%2Ftest%2F?query=%2Ftest%2Ftest%2F", nil) - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "/path/%2Ftest%2Ftest%2F", &setting.Cfg{}, httpClientProvider) require.NoError(t, err) proxy.HandleRequest() @@ -685,7 +680,7 @@ func TestNewDataSourceProxy_InvalidURL(t *testing.T) { } cfg := setting.Cfg{} plugin := plugins.DataSourcePlugin{} - _, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg) + _, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider()) require.Error(t, err) assert.True(t, strings.HasPrefix(err.Error(), `validation of data source URL "://host/root" failed`)) } @@ -704,7 +699,7 @@ func TestNewDataSourceProxy_ProtocolLessURL(t *testing.T) { cfg := setting.Cfg{} plugin := plugins.DataSourcePlugin{} - _, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg) + _, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider()) require.NoError(t, err) } @@ -744,7 +739,7 @@ func TestNewDataSourceProxy_MSSQL(t *testing.T) { Url: tc.url, } - p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg) + p, err := NewDataSourceProxy(&ds, &plugin, &ctx, "api/method", &cfg, httpclient.NewProvider()) if tc.err == nil { require.NoError(t, err) assert.Equal(t, &url.URL{ @@ -782,7 +777,7 @@ func getDatasourceProxiedRequest(t *testing.T, ctx *models.ReqContext, cfg *sett Url: "http://host/root/", } - proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg) + proxy, err := NewDataSourceProxy(ds, plugin, ctx, "", cfg, httpclient.NewProvider()) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) require.NoError(t, err) @@ -840,6 +835,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri test := &testCase{ datasource: &models.DataSource{ + Id: 1, Type: dsType, JsonData: simplejson.New(), }, @@ -892,7 +888,7 @@ func createAuthTest(t *testing.T, dsType string, authType string, authCheck stri func runDatasourceAuthTest(t *testing.T, test *testCase) { plugin := &plugins.DataSourcePlugin{} ctx := &models.ReqContext{} - proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(test.datasource, plugin, ctx, "", &setting.Cfg{}, httpclient.NewProvider()) require.NoError(t, err) req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil) @@ -933,7 +929,7 @@ func Test_PathCheck(t *testing.T) { return ctx, req } ctx, _ := setUp() - proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{}) + proxy, err := NewDataSourceProxy(&models.DataSource{}, plugin, ctx, "b", &setting.Cfg{}, httpclient.NewProvider()) require.NoError(t, err) require.Nil(t, proxy.validateRequest()) diff --git a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go new file mode 100644 index 00000000000..cdf2addcf11 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware.go @@ -0,0 +1,105 @@ +package httpclientprovider + +import ( + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/metrics/metricutil" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var datasourceRequestCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "grafana", + Name: "datasource_request_total", + Help: "A counter for outgoing requests for a datasource", + }, + []string{"datasource", "code", "method"}, +) + +var datasourceRequestSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "grafana", + Name: "datasource_request_duration_seconds", + Help: "summary of outgoing datasource requests sent from Grafana", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, []string{"datasource", "code", "method"}, +) + +var datasourceResponseSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "grafana", + Name: "datasource_response_size_bytes", + Help: "summary of datasource response sizes returned to Grafana", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, []string{"datasource"}, +) + +var datasourceRequestsInFlight = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "grafana", + Name: "datasource_request_in_flight", + Help: "A gauge of outgoing datasource requests currently being sent by Grafana", + }, + []string{"datasource"}, +) + +func init() { + prometheus.MustRegister(datasourceRequestSummary, + datasourceRequestCounter, + datasourceRequestsInFlight, + datasourceResponseSummary) +} + +const DataSourceMetricsMiddlewareName = "metrics" + +var executeMiddlewareFunc = executeMiddleware + +func DataSourceMetricsMiddleware() httpclient.Middleware { + return httpclient.NamedMiddlewareFunc(DataSourceMetricsMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { + if opts.Labels == nil { + return next + } + + datasourceName, exists := opts.Labels["datasource_name"] + if !exists { + return next + } + + datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName) + // if the datasource named cannot be turned into a prometheus + // label we will skip instrumenting these metrics. + if err != nil { + return next + } + + datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName} + + return executeMiddlewareFunc(next, datasourceLabel) + }) +} + +func executeMiddleware(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel) + requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel) + requestInFlight := datasourceRequestsInFlight.With(datasourceLabel) + responseSizeSummary := datasourceResponseSummary.With(datasourceLabel) + + res, err := promhttp.InstrumentRoundTripperDuration(requestSummary, + promhttp.InstrumentRoundTripperCounter(requestCounter, + promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))). + RoundTrip(r) + if err != nil { + return nil, err + } + // we avoid measuring contentlength less than zero because it indicates + // that the content size is unknown. https://godoc.org/github.com/badu/http#Response + if res != nil && res.ContentLength > 0 { + responseSizeSummary.Observe(float64(res.ContentLength)) + } + + return res, nil + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go new file mode 100644 index 00000000000..905ad182cab --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/datasource_metrics_middleware_test.go @@ -0,0 +1,130 @@ +package httpclientprovider + +import ( + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestDataSourceMetricsMiddleware(t *testing.T) { + t.Run("Without label options set should return next http.RoundTripper", func(t *testing.T) { + origExecuteMiddlewareFunc := executeMiddlewareFunc + executeMiddlewareCalled := false + middlewareCalled := false + executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { + executeMiddlewareCalled = true + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + middlewareCalled = true + return next.RoundTrip(r) + }) + } + t.Cleanup(func() { + executeMiddlewareFunc = origExecuteMiddlewareFunc + }) + + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("finalrt") + mw := DataSourceMetricsMiddleware() + rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) + require.False(t, executeMiddlewareCalled) + require.False(t, middlewareCalled) + }) + + t.Run("Without data source name label options set should return next http.RoundTripper", func(t *testing.T) { + origExecuteMiddlewareFunc := executeMiddlewareFunc + executeMiddlewareCalled := false + middlewareCalled := false + executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { + executeMiddlewareCalled = true + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + middlewareCalled = true + return next.RoundTrip(r) + }) + } + t.Cleanup(func() { + executeMiddlewareFunc = origExecuteMiddlewareFunc + }) + + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("finalrt") + mw := DataSourceMetricsMiddleware() + rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"test": "test"}}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) + require.False(t, executeMiddlewareCalled) + require.False(t, middlewareCalled) + }) + + t.Run("With datasource name label options set should execute middleware", func(t *testing.T) { + origExecuteMiddlewareFunc := executeMiddlewareFunc + executeMiddlewareCalled := false + datasourceLabels := prometheus.Labels{} + middlewareCalled := false + executeMiddlewareFunc = func(next http.RoundTripper, datasourceLabel prometheus.Labels) http.RoundTripper { + executeMiddlewareCalled = true + datasourceLabels = datasourceLabel + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + middlewareCalled = true + return next.RoundTrip(r) + }) + } + t.Cleanup(func() { + executeMiddlewareFunc = origExecuteMiddlewareFunc + }) + + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("finalrt") + mw := DataSourceMetricsMiddleware() + rt := mw.CreateMiddleware(httpclient.Options{Labels: map[string]string{"datasource_name": "My Data Source 123"}}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, DataSourceMetricsMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) + require.True(t, executeMiddlewareCalled) + require.Len(t, datasourceLabels, 1) + require.Equal(t, "My_Data_Source_123", datasourceLabels["datasource"]) + require.True(t, middlewareCalled) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go new file mode 100644 index 00000000000..61cb6836e22 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go @@ -0,0 +1,30 @@ +package httpclientprovider + +import ( + "fmt" + + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/infra/httpclient" + "github.com/grafana/grafana/pkg/setting" +) + +var newProviderFunc = sdkhttpclient.NewProvider + +// New creates a new HTTP client provider with pre-configured middlewares. +func New(cfg *setting.Cfg) httpclient.Provider { + userAgent := fmt.Sprintf("Grafana/%s", cfg.BuildVersion) + middlewares := []sdkhttpclient.Middleware{ + DataSourceMetricsMiddleware(), + SetUserAgentMiddleware(userAgent), + sdkhttpclient.BasicAuthenticationMiddleware(), + sdkhttpclient.CustomHeadersMiddleware(), + } + + if cfg.SigV4AuthEnabled { + middlewares = append(middlewares, SigV4Middleware()) + } + + return newProviderFunc(sdkhttpclient.ProviderOptions{ + Middlewares: middlewares, + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go new file mode 100644 index 00000000000..d5359eaf567 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go @@ -0,0 +1,52 @@ +package httpclientprovider + +import ( + "testing" + + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/require" +) + +func TestHTTPClientProvider(t *testing.T) { + t.Run("When creating new provider and SigV4 is disabled should apply expected middleware", func(t *testing.T) { + origNewProviderFunc := newProviderFunc + providerOpts := []sdkhttpclient.ProviderOptions{} + newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider { + providerOpts = opts + return nil + } + t.Cleanup(func() { + newProviderFunc = origNewProviderFunc + }) + _ = New(&setting.Cfg{SigV4AuthEnabled: false}) + require.Len(t, providerOpts, 1) + o := providerOpts[0] + require.Len(t, o.Middlewares, 4) + require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) + }) + + t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) { + origNewProviderFunc := newProviderFunc + providerOpts := []sdkhttpclient.ProviderOptions{} + newProviderFunc = func(opts ...sdkhttpclient.ProviderOptions) *sdkhttpclient.Provider { + providerOpts = opts + return nil + } + t.Cleanup(func() { + newProviderFunc = origNewProviderFunc + }) + _ = New(&setting.Cfg{SigV4AuthEnabled: true}) + require.Len(t, providerOpts, 1) + o := providerOpts[0] + require.Len(t, o.Middlewares, 5) + require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName()) + require.Equal(t, SigV4MiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName()) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go b/pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go new file mode 100644 index 00000000000..f22b083a122 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/sigv4_middleware.go @@ -0,0 +1,36 @@ +package httpclientprovider + +import ( + "net/http" + + "github.com/grafana/grafana-aws-sdk/pkg/sigv4" + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" +) + +// SigV4MiddlewareName the middleware name used by SigV4Middleware. +const SigV4MiddlewareName = "sigv4" + +var newSigV4Func = sigv4.New + +// SigV4Middleware applies AWS Signature Version 4 request signing for the outgoing request. +func SigV4Middleware() httpclient.Middleware { + return httpclient.NamedMiddlewareFunc(SigV4MiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { + if opts.SigV4 == nil { + return next + } + + return newSigV4Func( + &sigv4.Config{ + Service: opts.SigV4.Service, + AccessKey: opts.SigV4.AccessKey, + SecretKey: opts.SigV4.SecretKey, + Region: opts.SigV4.Region, + AssumeRoleARN: opts.SigV4.AssumeRoleARN, + AuthType: opts.SigV4.AuthType, + ExternalID: opts.SigV4.ExternalID, + Profile: opts.SigV4.Profile, + }, + next, + ) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go new file mode 100644 index 00000000000..c2eee03c9b9 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go @@ -0,0 +1,89 @@ +package httpclientprovider + +import ( + "net/http" + "testing" + + "github.com/grafana/grafana-aws-sdk/pkg/sigv4" + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/stretchr/testify/require" +) + +func TestSigV4Middleware(t *testing.T) { + t.Run("Without sigv4 options set should return next http.RoundTripper", func(t *testing.T) { + origSigV4Func := newSigV4Func + newSigV4Called := false + middlewareCalled := false + newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper { + newSigV4Called = true + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + middlewareCalled = true + return next.RoundTrip(r) + }) + } + t.Cleanup(func() { + newSigV4Func = origSigV4Func + }) + + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("finalrt") + mw := SigV4Middleware() + rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) + require.False(t, newSigV4Called) + require.False(t, middlewareCalled) + }) + + t.Run("With sigv4 options set should call sigv4 http.RoundTripper", func(t *testing.T) { + origSigV4Func := newSigV4Func + newSigV4Called := false + middlewareCalled := false + newSigV4Func = func(config *sigv4.Config, next http.RoundTripper) http.RoundTripper { + newSigV4Called = true + return httpclient.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + middlewareCalled = true + return next.RoundTrip(r) + }) + } + t.Cleanup(func() { + newSigV4Func = origSigV4Func + }) + + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("final") + mw := SigV4Middleware() + rt := mw.CreateMiddleware(httpclient.Options{SigV4: &httpclient.SigV4Config{}}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, SigV4MiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"final"}, ctx.callChain) + + require.True(t, newSigV4Called) + require.True(t, middlewareCalled) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/testing.go b/pkg/infra/httpclient/httpclientprovider/testing.go new file mode 100644 index 00000000000..d729c02d563 --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/testing.go @@ -0,0 +1,18 @@ +package httpclientprovider + +import ( + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" +) + +type testContext struct { + callChain []string +} + +func (c *testContext) createRoundTripper(name string) http.RoundTripper { + return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + c.callChain = append(c.callChain, name) + return &http.Response{StatusCode: http.StatusOK}, nil + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go b/pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go new file mode 100644 index 00000000000..18c382871da --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/user_agent_middleware.go @@ -0,0 +1,27 @@ +package httpclientprovider + +import ( + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" +) + +// SetUserAgentMiddlewareName is the middleware name used by SetUserAgentMiddleware. +const SetUserAgentMiddlewareName = "user-agent" + +// SetUserAgentMiddleware is middleware that sets the HTTP header User-Agent on the outgoing request. +// If User-Agent already set, it will not be overridden by this middleware. +func SetUserAgentMiddleware(userAgent string) httpclient.Middleware { + return httpclient.NamedMiddlewareFunc(SetUserAgentMiddlewareName, func(opts httpclient.Options, next http.RoundTripper) http.RoundTripper { + if userAgent == "" { + return next + } + + return httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", userAgent) + } + return next.RoundTrip(req) + }) + }) +} diff --git a/pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go new file mode 100644 index 00000000000..9aee212c06b --- /dev/null +++ b/pkg/infra/httpclient/httpclientprovider/user_agent_middleware_test.go @@ -0,0 +1,82 @@ +package httpclientprovider + +import ( + "net/http" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/stretchr/testify/require" +) + +func TestCustomHeadersMiddleware(t *testing.T) { + t.Run("Without user agent set should return next http.RoundTripper", func(t *testing.T) { + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("finalrt") + mw := SetUserAgentMiddleware("") + rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"finalrt"}, ctx.callChain) + }) + + t.Run("With user agent set should apply HTTP headers to the request", func(t *testing.T) { + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("final") + mw := SetUserAgentMiddleware("Grafana/8.0.0") + rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"final"}, ctx.callChain) + + require.Equal(t, "Grafana/8.0.0", req.Header.Get("User-Agent")) + }) + + t.Run("With user agent set, but request already has User-Agent header set should not apply HTTP headers to the request", func(t *testing.T) { + ctx := &testContext{} + finalRoundTripper := ctx.createRoundTripper("final") + mw := SetUserAgentMiddleware("Grafana/8.0.0") + rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper) + require.NotNil(t, rt) + middlewareName, ok := mw.(httpclient.MiddlewareName) + require.True(t, ok) + require.Equal(t, SetUserAgentMiddlewareName, middlewareName.MiddlewareName()) + + req, err := http.NewRequest(http.MethodGet, "http://", nil) + require.NoError(t, err) + req.Header.Set("User-Agent", "ua") + res, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NotNil(t, res) + if res.Body != nil { + require.NoError(t, res.Body.Close()) + } + require.Len(t, ctx.callChain, 1) + require.ElementsMatch(t, []string{"final"}, ctx.callChain) + + require.Equal(t, "ua", req.Header.Get("User-Agent")) + }) +} diff --git a/pkg/infra/httpclient/provider.go b/pkg/infra/httpclient/provider.go new file mode 100644 index 00000000000..8b9df15dbdf --- /dev/null +++ b/pkg/infra/httpclient/provider.go @@ -0,0 +1,32 @@ +package httpclient + +import ( + "crypto/tls" + "net/http" + + "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" +) + +// Provider provides abilities to create http.Client, http.RoundTripper and tls.Config. +type Provider interface { + // New creates a new http.Client given provided options. + New(opts ...httpclient.Options) (*http.Client, error) + + // GetTransport creates a new http.RoundTripper given provided options. + GetTransport(opts ...httpclient.Options) (http.RoundTripper, error) + + // GetTLSConfig creates a new tls.Config given provided options. + GetTLSConfig(opts ...httpclient.Options) (*tls.Config, error) +} + +// NewProvider creates a new HTTP client provider. +// Optionally provide ProviderOptions options that will be used as default if +// not specified in Options argument to Provider.New, Provider.GetTransport and +// Provider.GetTLSConfig. +// If no middlewares are provided in opts the DefaultMiddlewares() will be used. If you +// provide middlewares you have to manually add the DefaultMiddlewares() for it to be +// enabled. +// Note: Middlewares will be executed in the same order as provided. +func NewProvider(opts ...httpclient.ProviderOptions) *httpclient.Provider { + return httpclient.NewProvider(opts...) +} diff --git a/pkg/models/datasource_cache.go b/pkg/models/datasource_cache.go index 76b92308b09..292c74560ef 100644 --- a/pkg/models/datasource_cache.go +++ b/pkg/models/datasource_cache.go @@ -2,128 +2,17 @@ package models import ( "crypto/tls" - "crypto/x509" - "errors" "fmt" - "net" "net/http" "sync" "time" - "github.com/grafana/grafana-aws-sdk/pkg/sigv4" - - "github.com/grafana/grafana/pkg/infra/metrics/metricutil" + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/setting" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -var datasourceRequestCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: "grafana", - Name: "datasource_request_total", - Help: "A counter for outgoing requests for a datasource", - }, - []string{"datasource", "code", "method"}, -) - -var datasourceRequestSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "grafana", - Name: "datasource_request_duration_seconds", - Help: "summary of outgoing datasource requests sent from Grafana", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, []string{"datasource", "code", "method"}, -) - -var datasourceResponseSummary = prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: "grafana", - Name: "datasource_response_size_bytes", - Help: "summary of datasource response sizes returned to Grafana", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, []string{"datasource"}, -) - -var datasourceRequestsInFlight = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "grafana", - Name: "datasource_request_in_flight", - Help: "A gauge of outgoing datasource requests currently being sent by Grafana", - }, - []string{"datasource"}, ) -func init() { - prometheus.MustRegister(datasourceRequestSummary, - datasourceRequestCounter, - datasourceRequestsInFlight, - datasourceResponseSummary) -} - -type proxyTransportCache struct { - cache map[int64]cachedTransport - sync.Mutex -} - -// dataSourceTransport implements http.RoundTripper (https://golang.org/pkg/net/http/#RoundTripper) -type dataSourceTransport struct { - datasourceName string - headers map[string]string - transport *http.Transport - next http.RoundTripper -} - -func instrumentRoundtrip(datasourceName string, next http.RoundTripper) promhttp.RoundTripperFunc { - return promhttp.RoundTripperFunc(func(r *http.Request) (*http.Response, error) { - datasourceLabelName, err := metricutil.SanitizeLabelName(datasourceName) - // if the datasource named cannot be turned into a prometheus - // label we will skip instrumenting these metrics. - if err != nil { - return next.RoundTrip(r) - } - - datasourceLabel := prometheus.Labels{"datasource": datasourceLabelName} - - requestCounter := datasourceRequestCounter.MustCurryWith(datasourceLabel) - requestSummary := datasourceRequestSummary.MustCurryWith(datasourceLabel) - requestInFlight := datasourceRequestsInFlight.With(datasourceLabel) - responseSizeSummary := datasourceResponseSummary.With(datasourceLabel) - - res, err := promhttp.InstrumentRoundTripperDuration(requestSummary, - promhttp.InstrumentRoundTripperCounter(requestCounter, - promhttp.InstrumentRoundTripperInFlight(requestInFlight, next))). - RoundTrip(r) - - // we avoid measuring contentlength less than zero because it indicates - // that the content size is unknown. https://godoc.org/github.com/badu/http#Response - if res != nil && res.ContentLength > 0 { - responseSizeSummary.Observe(float64(res.ContentLength)) - } - - return res, err - }) -} - -// RoundTrip executes a single HTTP transaction, returning a Response for the provided Request. -func (d *dataSourceTransport) RoundTrip(req *http.Request) (*http.Response, error) { - for key, value := range d.headers { - req.Header.Set(key, value) - } - - return instrumentRoundtrip(d.datasourceName, d.next).RoundTrip(req) -} - -type cachedTransport struct { - updated time.Time - - *dataSourceTransport -} - -var ptc = proxyTransportCache{ - cache: make(map[int64]cachedTransport), -} - func (ds *DataSource) getTimeout() time.Duration { timeout := 0 if ds.JsonData != nil { @@ -135,8 +24,22 @@ func (ds *DataSource) getTimeout() time.Duration { return time.Duration(timeout) * time.Second } -func (ds *DataSource) GetHttpClient() (*http.Client, error) { - transport, err := ds.GetHttpTransport() +type proxyTransportCache struct { + cache map[int64]cachedRoundTripper + sync.Mutex +} + +type cachedRoundTripper struct { + updated time.Time + roundTripper http.RoundTripper +} + +var ptc = proxyTransportCache{ + cache: make(map[int64]cachedRoundTripper), +} + +func (ds *DataSource) GetHTTPClient(provider httpclient.Provider) (*http.Client, error) { + transport, err := ds.GetHTTPTransport(provider) if err != nil { return nil, err } @@ -147,79 +50,86 @@ func (ds *DataSource) GetHttpClient() (*http.Client, error) { }, nil } -// Creates a HTTP Transport middleware chain -func (ds *DataSource) GetHttpTransport() (*dataSourceTransport, error) { +func (ds *DataSource) GetHTTPTransport(provider httpclient.Provider) (http.RoundTripper, error) { ptc.Lock() defer ptc.Unlock() if t, present := ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) { - return t.dataSourceTransport, nil + return t.roundTripper, nil } - tlsConfig, err := ds.GetTLSConfig() + rt, err := provider.GetTransport(ds.HTTPClientOptions()) if err != nil { return nil, err } - tlsConfig.Renegotiation = tls.RenegotiateFreelyAsClient - - // Create transport which adds all - customHeaders := ds.getCustomHeaders() - transport := &http.Transport{ - TLSClientConfig: tlsConfig, - Proxy: http.ProxyFromEnvironment, - Dial: (&net.Dialer{ - Timeout: ds.getTimeout(), - KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second, - }).Dial, - TLSHandshakeTimeout: time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second, - ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second, - MaxIdleConns: setting.DataProxyMaxIdleConns, - IdleConnTimeout: time.Duration(setting.DataProxyIdleConnTimeout) * time.Second, + ptc.cache[ds.Id] = cachedRoundTripper{ + roundTripper: rt, + updated: ds.Updated, } - // Set default next round tripper to the default transport - next := http.RoundTripper(transport) + return rt, nil +} - // Add SigV4 middleware if enabled, which will then defer to the default transport - if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool() && setting.SigV4AuthEnabled { - next = ds.sigV4Middleware(transport) +func (ds *DataSource) HTTPClientOptions() sdkhttpclient.Options { + tlsOptions := ds.TLSOptions() + opts := sdkhttpclient.Options{ + Timeouts: &sdkhttpclient.TimeoutOptions{ + Timeout: ds.getTimeout(), + KeepAlive: time.Duration(setting.DataProxyKeepAlive) * time.Second, + TLSHandshakeTimeout: time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second, + ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second, + MaxIdleConns: setting.DataProxyMaxIdleConns, + MaxIdleConnsPerHost: setting.DataProxyMaxIdleConnsPerHost, + IdleConnTimeout: time.Duration(setting.DataProxyIdleConnTimeout) * time.Second, + }, + Headers: getCustomHeaders(ds.JsonData, ds.DecryptedValues()), + Labels: map[string]string{ + "datasource_name": ds.Name, + "datasource_uid": ds.Uid, + }, + TLS: &tlsOptions, } - dsTransport := &dataSourceTransport{ - datasourceName: ds.Name, - headers: customHeaders, - transport: transport, - next: next, + if ds.JsonData != nil { + opts.CustomOptions = ds.JsonData.MustMap() } - ptc.cache[ds.Id] = cachedTransport{ - dataSourceTransport: dsTransport, - updated: ds.Updated, + if ds.BasicAuth { + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ + User: ds.BasicAuthUser, + Password: ds.DecryptedBasicAuthPassword(), + } + } else if ds.User != "" { + opts.BasicAuth = &sdkhttpclient.BasicAuthOptions{ + User: ds.User, + Password: ds.DecryptedPassword(), + } } - return dsTransport, nil -} - -func (ds *DataSource) sigV4Middleware(next http.RoundTripper) http.RoundTripper { - decrypted := ds.DecryptedValues() - - return sigv4.New( - &sigv4.Config{ + if ds.JsonData != nil && ds.JsonData.Get("sigV4Auth").MustBool(false) { + opts.SigV4 = &sdkhttpclient.SigV4Config{ Service: awsServiceNamespace(ds.Type), - AccessKey: decrypted["sigV4AccessKey"], - SecretKey: decrypted["sigV4SecretKey"], Region: ds.JsonData.Get("sigV4Region").MustString(), AssumeRoleARN: ds.JsonData.Get("sigV4AssumeRoleArn").MustString(), AuthType: ds.JsonData.Get("sigV4AuthType").MustString(), ExternalID: ds.JsonData.Get("sigV4ExternalId").MustString(), Profile: ds.JsonData.Get("sigV4Profile").MustString(), - }, - next, - ) + } + + if val, exists := ds.DecryptedValue("sigV4AccessKey"); exists { + opts.SigV4.AccessKey = val + } + + if val, exists := ds.DecryptedValue("sigV4SecretKey"); exists { + opts.SigV4.SecretKey = val + } + } + + return opts } -func (ds *DataSource) GetTLSConfig() (*tls.Config, error) { +func (ds *DataSource) TLSOptions() sdkhttpclient.TLSOptions { var tlsSkipVerify, tlsClientAuth, tlsAuthWithCACert bool var serverName string @@ -230,55 +140,55 @@ func (ds *DataSource) GetTLSConfig() (*tls.Config, error) { serverName = ds.JsonData.Get("serverName").MustString() } - tlsConfig := &tls.Config{ + opts := sdkhttpclient.TLSOptions{ InsecureSkipVerify: tlsSkipVerify, ServerName: serverName, } if tlsClientAuth || tlsAuthWithCACert { - decrypted := ds.SecureJsonData.Decrypt() - if tlsAuthWithCACert && len(decrypted["tlsCACert"]) > 0 { - caPool := x509.NewCertPool() - ok := caPool.AppendCertsFromPEM([]byte(decrypted["tlsCACert"])) - if !ok { - return nil, errors.New("failed to parse TLS CA PEM certificate") + if tlsAuthWithCACert { + if val, exists := ds.DecryptedValue("tlsCACert"); exists && len(val) > 0 { + opts.CACertificate = val } - tlsConfig.RootCAs = caPool } if tlsClientAuth { - cert, err := tls.X509KeyPair([]byte(decrypted["tlsClientCert"]), []byte(decrypted["tlsClientKey"])) - if err != nil { - return nil, err + if val, exists := ds.DecryptedValue("tlsClientCert"); exists && len(val) > 0 { + opts.ClientCertificate = val + } + if val, exists := ds.DecryptedValue("tlsClientKey"); exists && len(val) > 0 { + opts.ClientKey = val } - tlsConfig.Certificates = []tls.Certificate{cert} } } - return tlsConfig, nil + return opts +} + +func (ds *DataSource) GetTLSConfig(httpClientProvider httpclient.Provider) (*tls.Config, error) { + return httpClientProvider.GetTLSConfig(ds.HTTPClientOptions()) } // getCustomHeaders returns a map with all the to be set headers // The map key represents the HeaderName and the value represents this header's value -func (ds *DataSource) getCustomHeaders() map[string]string { +func getCustomHeaders(jsonData *simplejson.Json, decryptedValues map[string]string) map[string]string { headers := make(map[string]string) - if ds.JsonData == nil { + if jsonData == nil { return headers } - decrypted := ds.SecureJsonData.Decrypt() index := 1 for { headerNameSuffix := fmt.Sprintf("httpHeaderName%d", index) headerValueSuffix := fmt.Sprintf("httpHeaderValue%d", index) - key := ds.JsonData.Get(headerNameSuffix).MustString() + key := jsonData.Get(headerNameSuffix).MustString() if key == "" { // No (more) header values are available break } - if val, ok := decrypted[headerValueSuffix]; ok { + if val, ok := decryptedValues[headerValueSuffix]; ok { headers[key] = val } index++ diff --git a/pkg/models/datasource_cache_test.go b/pkg/models/datasource_cache_test.go index 9e4a930e519..9569d9c3dd6 100644 --- a/pkg/models/datasource_cache_test.go +++ b/pkg/models/datasource_cache_test.go @@ -8,17 +8,25 @@ import ( "testing" "time" + sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/util" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) //nolint:goconst func TestDataSource_GetHttpTransport(t *testing.T) { t.Run("Should use cached proxy", func(t *testing.T) { + var configuredTransport *http.Transport + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredTransport = transport + }, + }) + clearDSProxyCache(t) ds := DataSource{ Id: 1, @@ -26,20 +34,30 @@ func TestDataSource_GetHttpTransport(t *testing.T) { Type: "Kubernetes", } - tr1, err := ds.GetHttpTransport() + rt1, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt1) + tr1 := configuredTransport - tr2, err := ds.GetHttpTransport() + rt2, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt2) + tr2 := configuredTransport require.Same(t, tr1, tr2) - assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify) - assert.Empty(t, tr1.transport.TLSClientConfig.Certificates) - assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs) + require.False(t, tr1.TLSClientConfig.InsecureSkipVerify) + require.Empty(t, tr1.TLSClientConfig.Certificates) + require.Nil(t, tr1.TLSClientConfig.RootCAs) }) t.Run("Should not use cached proxy when datasource updated", func(t *testing.T) { + var configuredTransport *http.Transport + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredTransport = transport + }, + }) clearDSProxyCache(t) setting.SecretKey = "password" @@ -56,26 +74,36 @@ func TestDataSource_GetHttpTransport(t *testing.T) { Updated: time.Now().Add(-2 * time.Minute), } - tr1, err := ds.GetHttpTransport() + rt1, err := ds.GetHTTPTransport(provider) + require.NotNil(t, rt1) require.NoError(t, err) - assert.False(t, tr1.transport.TLSClientConfig.InsecureSkipVerify) - assert.Empty(t, tr1.transport.TLSClientConfig.Certificates) - assert.Nil(t, tr1.transport.TLSClientConfig.RootCAs) + tr1 := configuredTransport + + require.False(t, tr1.TLSClientConfig.InsecureSkipVerify) + require.Empty(t, tr1.TLSClientConfig.Certificates) + require.Nil(t, tr1.TLSClientConfig.RootCAs) ds.JsonData = nil ds.SecureJsonData = map[string][]byte{} ds.Updated = time.Now() - tr2, err := ds.GetHttpTransport() + rt2, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt2) + tr2 := configuredTransport require.NotSame(t, tr1, tr2) - - assert.Nil(t, tr2.transport.TLSClientConfig.RootCAs) + require.Nil(t, tr2.TLSClientConfig.RootCAs) }) t.Run("Should set TLS client authentication enabled if configured in JsonData", func(t *testing.T) { + var configuredTransport *http.Transport + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredTransport = transport + }, + }) clearDSProxyCache(t) setting.SecretKey = "password" @@ -98,15 +126,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) { }, } - tr, err := ds.GetHttpTransport() + rt, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt) + tr := configuredTransport - assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify) - require.Len(t, tr.transport.TLSClientConfig.Certificates, 1) + require.False(t, tr.TLSClientConfig.InsecureSkipVerify) + require.Len(t, tr.TLSClientConfig.Certificates, 1) }) t.Run("Should set user-supplied TLS CA if configured in JsonData", func(t *testing.T) { + var configuredTransport *http.Transport + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredTransport = transport + }, + }) clearDSProxyCache(t) + ClearDSDecryptionCache() setting.SecretKey = "password" json := simplejson.New() @@ -126,15 +163,23 @@ func TestDataSource_GetHttpTransport(t *testing.T) { }, } - tr, err := ds.GetHttpTransport() + rt, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt) + tr := configuredTransport - assert.False(t, tr.transport.TLSClientConfig.InsecureSkipVerify) - require.Len(t, tr.transport.TLSClientConfig.RootCAs.Subjects(), 1) - assert.Equal(t, "server-name", tr.transport.TLSClientConfig.ServerName) + require.False(t, tr.TLSClientConfig.InsecureSkipVerify) + require.Len(t, tr.TLSClientConfig.RootCAs.Subjects(), 1) + require.Equal(t, "server-name", tr.TLSClientConfig.ServerName) }) t.Run("Should set skip TLS verification if configured in JsonData", func(t *testing.T) { + var configuredTransport *http.Transport + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredTransport = transport + }, + }) clearDSProxyCache(t) json := simplejson.New() @@ -147,19 +192,24 @@ func TestDataSource_GetHttpTransport(t *testing.T) { JsonData: json, } - tr1, err := ds.GetHttpTransport() + rt1, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt1) + tr1 := configuredTransport - tr2, err := ds.GetHttpTransport() + rt2, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt2) + tr2 := configuredTransport require.Same(t, tr1, tr2) - - assert.True(t, tr1.transport.TLSClientConfig.InsecureSkipVerify) + require.True(t, tr1.TLSClientConfig.InsecureSkipVerify) }) t.Run("Should set custom headers if configured in JsonData", func(t *testing.T) { + provider := httpclient.NewProvider() clearDSProxyCache(t) + ClearDSDecryptionCache() json := simplejson.NewFromAny(map[string]interface{}{ "httpHeaderName1": "Authorization", @@ -177,8 +227,8 @@ func TestDataSource_GetHttpTransport(t *testing.T) { SecureJsonData: map[string][]byte{"httpHeaderValue1": encryptedData}, } - headers := ds.getCustomHeaders() - assert.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"]) + headers := getCustomHeaders(json, map[string]string{"httpHeaderValue1": "Bearer xf5yhfkpsnmgo"}) + require.Equal(t, "Bearer xf5yhfkpsnmgo", headers["Authorization"]) // 1. Start HTTP test server which checks the request headers backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -197,12 +247,13 @@ func TestDataSource_GetHttpTransport(t *testing.T) { // 2. Get HTTP transport from datasource which uses the test server as backend ds.Url = backend.URL - tr, err := ds.GetHttpTransport() + rt, err := ds.GetHTTPTransport(provider) require.NoError(t, err) + require.NotNil(t, rt) // 3. Send test request which should have the Authorization header set req := httptest.NewRequest("GET", backend.URL+"/test-headers", nil) - res, err := tr.RoundTrip(req) + res, err := rt.RoundTrip(req) require.NoError(t, err) t.Cleanup(func() { err := res.Body.Close() @@ -211,10 +262,11 @@ func TestDataSource_GetHttpTransport(t *testing.T) { body, err := ioutil.ReadAll(res.Body) require.NoError(t, err) bodyStr := string(body) - assert.Equal(t, "Ok", bodyStr) + require.Equal(t, "Ok", bodyStr) }) t.Run("Should use request timeout if configured in JsonData", func(t *testing.T) { + provider := httpclient.NewProvider() clearDSProxyCache(t) json := simplejson.NewFromAny(map[string]interface{}{ @@ -227,47 +279,34 @@ func TestDataSource_GetHttpTransport(t *testing.T) { JsonData: json, } - client, err := ds.GetHttpClient() + client, err := ds.GetHTTPClient(provider) require.NoError(t, err) - - assert.Equal(t, 19*time.Second, client.Timeout) + require.NotNil(t, client) + require.Equal(t, 19*time.Second, client.Timeout) }) - t.Run("Should not include SigV4 middleware if not configured in JsonData", func(t *testing.T) { - clearDSProxyCache(t) - - origEnabled := setting.SigV4AuthEnabled - setting.SigV4AuthEnabled = true - t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled }) - - ds := DataSource{} - - tr, err := ds.GetHttpTransport() - require.NoError(t, err) - - _, ok := tr.next.(*http.Transport) - require.True(t, ok) - }) - - t.Run("Should not include SigV4 middleware if not configured in app config", func(t *testing.T) { + t.Run("Should populate SigV4 options if configured in JsonData", func(t *testing.T) { + var configuredOpts sdkhttpclient.Options + provider := httpclient.NewProvider(sdkhttpclient.ProviderOptions{ + ConfigureTransport: func(opts sdkhttpclient.Options, transport *http.Transport) { + configuredOpts = opts + }, + }) clearDSProxyCache(t) - origEnabled := setting.SigV4AuthEnabled - setting.SigV4AuthEnabled = false - t.Cleanup(func() { setting.SigV4AuthEnabled = origEnabled }) - json, err := simplejson.NewJson([]byte(`{ "sigV4Auth": true }`)) require.NoError(t, err) ds := DataSource{ + Type: DS_ES, JsonData: json, } - tr, err := ds.GetHttpTransport() + _, err = ds.GetHTTPTransport(provider) require.NoError(t, err) - - _, ok := tr.next.(*http.Transport) - require.True(t, ok) + require.NotNil(t, configuredOpts) + require.NotNil(t, configuredOpts.SigV4) + require.Equal(t, "es", configuredOpts.SigV4.Service) }) } @@ -288,7 +327,7 @@ func TestDataSource_DecryptedValue(t *testing.T) { // Populate cache password, ok := ds.DecryptedValue("password") require.True(t, ok) - assert.Equal(t, "password", password) + require.Equal(t, "password", password) ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{ "password": "", @@ -296,7 +335,7 @@ func TestDataSource_DecryptedValue(t *testing.T) { password, ok = ds.DecryptedValue("password") require.True(t, ok) - assert.Equal(t, "password", password) + require.Equal(t, "password", password) }) t.Run("When datasource is updated, encrypted JSON should not be fetched from cache", func(t *testing.T) { @@ -315,7 +354,7 @@ func TestDataSource_DecryptedValue(t *testing.T) { // Populate cache password, ok := ds.DecryptedValue("password") require.True(t, ok) - assert.Equal(t, "password", password) + require.Equal(t, "password", password) ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{ "password": "", @@ -324,7 +363,7 @@ func TestDataSource_DecryptedValue(t *testing.T) { password, ok = ds.DecryptedValue("password") require.True(t, ok) - assert.Empty(t, password) + require.Empty(t, password) }) } @@ -334,7 +373,7 @@ func clearDSProxyCache(t *testing.T) { ptc.Lock() defer ptc.Unlock() - ptc.cache = make(map[int64]cachedTransport) + ptc.cache = make(map[int64]cachedRoundTripper) } const caCert string = `-----BEGIN CERTIFICATE----- diff --git a/pkg/server/server.go b/pkg/server/server.go index b3053e5d7b0..368951d27a2 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/bus" _ "github.com/grafana/grafana/pkg/extensions" + "github.com/grafana/grafana/pkg/infra/httpclient/httpclientprovider" "github.com/grafana/grafana/pkg/infra/localcache" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics" @@ -281,6 +282,7 @@ func (s *Server) buildServiceGraph(services []*registry.Descriptor) error { s.cfg, routing.NewRouteRegister(middleware.ProvideRouteOperationName, middleware.RequestMetrics(s.cfg)), localcache.New(5*time.Minute, 10*time.Minute), + httpclientprovider.New(s.cfg), s, } return registry.BuildServiceGraph(objs, services) diff --git a/pkg/services/datasourceproxy/datasourceproxy.go b/pkg/services/datasourceproxy/datasourceproxy.go index 87b0d3526fb..e7c62a4e1ff 100644 --- a/pkg/services/datasourceproxy/datasourceproxy.go +++ b/pkg/services/datasourceproxy/datasourceproxy.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/api/datasource" "github.com/grafana/grafana/pkg/api/pluginproxy" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -25,6 +26,7 @@ type DatasourceProxyService struct { PluginRequestValidator models.PluginRequestValidator `inject:""` PluginManager plugins.Manager `inject:""` Cfg *setting.Cfg `inject:""` + HTTPClientProvider httpclient.Provider `inject:""` } func (p *DatasourceProxyService) Init() error { @@ -62,7 +64,7 @@ func (p *DatasourceProxyService) ProxyDatasourceRequestWithID(c *models.ReqConte } proxyPath := getProxyPath(c) - proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg) + proxy, err := pluginproxy.NewDataSourceProxy(ds, plugin, c, proxyPath, p.Cfg, p.HTTPClientProvider) if err != nil { if errors.Is(err, datasource.URLValidationError{}) { c.JsonApiErr(http.StatusBadRequest, fmt.Sprintf("Invalid data source URL: %q", ds.Url), err) diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 3f3dc7eb87a..65df953c046 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -81,6 +81,7 @@ var ( DataProxyTLSHandshakeTimeout int DataProxyExpectContinueTimeout int DataProxyMaxIdleConns int + DataProxyMaxIdleConnsPerHost int DataProxyKeepAlive int DataProxyIdleConnTimeout int StaticRootPath string @@ -827,6 +828,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10) DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1) DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100) + DataProxyMaxIdleConnsPerHost = dataproxy.Key("max_idle_connections_per_host").MustInt(2) DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90) cfg.SendUserHeader = dataproxy.Key("send_user_header").MustBool(false) diff --git a/pkg/tsdb/azuremonitor/applicationinsights-datasource.go b/pkg/tsdb/azuremonitor/applicationinsights-datasource.go index 1a82c4ddfe5..46c4841deee 100644 --- a/pkg/tsdb/azuremonitor/applicationinsights-datasource.go +++ b/pkg/tsdb/azuremonitor/applicationinsights-datasource.go @@ -241,8 +241,6 @@ func (e *ApplicationInsightsDatasource) createRequest(ctx context.Context, dsInf return nil, errutil.Wrap("Failed to create request", err) } - req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg) return req, nil diff --git a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go index 80fded643ac..8d59df5f748 100644 --- a/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/azure-log-analytics-datasource.go @@ -221,7 +221,6 @@ func (e *AzureLogAnalyticsDatasource) createRequest(ctx context.Context, dsInfo } req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) // find plugin plugin := e.pluginManager.GetDataSource(dsInfo.Type) diff --git a/pkg/tsdb/azuremonitor/azuremonitor-datasource.go b/pkg/tsdb/azuremonitor/azuremonitor-datasource.go index 488afdc5af5..3a94b816335 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor-datasource.go +++ b/pkg/tsdb/azuremonitor/azuremonitor-datasource.go @@ -254,7 +254,6 @@ func (e *AzureMonitorDatasource) createRequest(ctx context.Context, dsInfo *mode } req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) pluginproxy.ApplyRoute(ctx, req, proxyPass, azureMonitorRoute, dsInfo, e.cfg) diff --git a/pkg/tsdb/azuremonitor/azuremonitor.go b/pkg/tsdb/azuremonitor/azuremonitor.go index 4d6a1b0331a..29841daa02f 100644 --- a/pkg/tsdb/azuremonitor/azuremonitor.go +++ b/pkg/tsdb/azuremonitor/azuremonitor.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -29,8 +30,9 @@ func init() { } type Service struct { - PluginManager plugins.Manager `inject:""` - Cfg *setting.Cfg `inject:""` + PluginManager plugins.Manager `inject:""` + HTTPClientProvider httpclient.Provider `inject:""` + Cfg *setting.Cfg `inject:""` } func (s *Service) Init() error { @@ -48,7 +50,7 @@ type AzureMonitorExecutor struct { // NewAzureMonitorExecutor initializes a http client //nolint: staticcheck // plugins.DataPlugin deprecated func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) { - httpClient, err := dsInfo.GetHttpClient() + httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider) if err != nil { return nil, err } diff --git a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go index 11137661b3d..93d94af09e2 100644 --- a/pkg/tsdb/azuremonitor/insights-analytics-datasource.go +++ b/pkg/tsdb/azuremonitor/insights-analytics-datasource.go @@ -215,8 +215,6 @@ func (e *InsightsAnalyticsDatasource) createRequest(ctx context.Context, dsInfo return nil, errutil.Wrap("Failed to create request", err) } - req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) - pluginproxy.ApplyRoute(ctx, req, proxyPass, appInsightsRoute, dsInfo, e.cfg) return req, nil diff --git a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go index 0b4ff46869d..16f16d78330 100644 --- a/pkg/tsdb/cloudmonitoring/cloudmonitoring.go +++ b/pkg/tsdb/cloudmonitoring/cloudmonitoring.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/api/pluginproxy" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/setting" @@ -75,8 +76,9 @@ func init() { } type Service struct { - PluginManager plugins.Manager `inject:""` - Cfg *setting.Cfg `inject:""` + PluginManager plugins.Manager `inject:""` + HTTPClientProvider httpclient.Provider `inject:""` + Cfg *setting.Cfg `inject:""` } func (s *Service) Init() error { @@ -94,7 +96,7 @@ type Executor struct { // NewExecutor returns an Executor. //nolint: staticcheck // plugins.DataPlugin deprecated func (s *Service) NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) { - httpClient, err := dsInfo.GetHttpClient() + httpClient, err := dsInfo.GetHTTPClient(s.HTTPClientProvider) if err != nil { return nil, err } @@ -540,7 +542,6 @@ func (e *Executor) createRequest(ctx context.Context, dsInfo *models.DataSource, } req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) // find plugin plugin := e.pluginManager.GetDataSource(dsInfo.Type) diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 1efc017dd73..4176c119ad3 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -15,6 +15,7 @@ import ( "github.com/Masterminds/semver" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/tsdb/interval" @@ -29,8 +30,8 @@ var ( clientLog = log.New(loggerName) ) -var newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { - return ds.GetHttpClient() +var newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) { + return ds.GetHTTPClient(httpClientProvider) } // Client represents a client which can interact with elasticsearch api @@ -72,7 +73,7 @@ func coerceVersion(v *simplejson.Json) (*semver.Version, error) { } // NewClient creates a new elasticsearch client -var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) { +var NewClient = func(ctx context.Context, httpClientProvider httpclient.Provider, ds *models.DataSource, timeRange plugins.DataTimeRange) (Client, error) { version, err := coerceVersion(ds.JsonData.Get("esVersion")) if err != nil { @@ -108,13 +109,14 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange plugi } type baseClientImpl struct { - ctx context.Context - ds *models.DataSource - version *semver.Version - timeField string - indices []string - timeRange plugins.DataTimeRange - debugEnabled bool + ctx context.Context + httpClientProvider httpclient.Provider + ds *models.DataSource + version *semver.Version + timeField string + indices []string + timeRange plugins.DataTimeRange + debugEnabled bool } func (c *baseClientImpl) GetVersion() *semver.Version { @@ -208,20 +210,9 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [ } } - req.Header.Set("User-Agent", "Grafana") req.Header.Set("Content-Type", "application/x-ndjson") - if c.ds.BasicAuth { - clientLog.Debug("Request configured to use basic authentication") - req.SetBasicAuth(c.ds.BasicAuthUser, c.ds.DecryptedBasicAuthPassword()) - } - - if !c.ds.BasicAuth && c.ds.User != "" { - clientLog.Debug("Request configured to use basic authentication") - req.SetBasicAuth(c.ds.User, c.ds.DecryptedPassword()) - } - - httpClient, err := newDatasourceHttpClient(c.ds) + httpClient, err := newDatasourceHttpClient(c.httpClientProvider, c.ds) if err != nil { return nil, err } diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index 6a933a4fc4a..61537d214a9 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/tsdb/interval" @@ -24,7 +25,7 @@ func TestNewClient(t *testing.T) { JsonData: simplejson.NewFromAny(make(map[string]interface{})), } - _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + _, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.Error(t, err) }) @@ -35,7 +36,7 @@ func TestNewClient(t *testing.T) { }), } - _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + _, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.Error(t, err) }) @@ -48,7 +49,7 @@ func TestNewClient(t *testing.T) { }), } - _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + _, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.Error(t, err) }) @@ -60,7 +61,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, "2.0.0", c.GetVersion().String()) }) @@ -73,7 +74,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, "5.0.0", c.GetVersion().String()) }) @@ -86,7 +87,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, "5.6.0", c.GetVersion().String()) }) @@ -99,7 +100,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, "6.0.0", c.GetVersion().String()) }) @@ -112,7 +113,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, "7.0.0", c.GetVersion().String()) }) @@ -127,7 +128,7 @@ func TestNewClient(t *testing.T) { }), } - c, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.NoError(t, err) assert.Equal(t, version, c.GetVersion().String()) }) @@ -141,7 +142,7 @@ func TestNewClient(t *testing.T) { }), } - _, err := NewClient(context.Background(), ds, plugins.DataTimeRange{}) + _, err := NewClient(context.Background(), httpclient.NewProvider(), ds, plugins.DataTimeRange{}) require.Error(t, err) }) } @@ -408,14 +409,14 @@ func httpClientScenario(t *testing.T, desc string, ds *models.DataSource, fn sce toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond)) timeRange := plugins.NewDataTimeRange(fromStr, toStr) - c, err := NewClient(context.Background(), ds, timeRange) + c, err := NewClient(context.Background(), httpclient.NewProvider(), ds, timeRange) require.NoError(t, err) require.NotNil(t, c) sc.client = c currentNewDatasourceHTTPClient := newDatasourceHttpClient - newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) { + newDatasourceHttpClient = func(httpClientProvider httpclient.Provider, ds *models.DataSource) (*http.Client, error) { return ts.Client(), nil } diff --git a/pkg/tsdb/elasticsearch/elasticsearch.go b/pkg/tsdb/elasticsearch/elasticsearch.go index b55660a8f5d..3496c0bf664 100644 --- a/pkg/tsdb/elasticsearch/elasticsearch.go +++ b/pkg/tsdb/elasticsearch/elasticsearch.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" es "github.com/grafana/grafana/pkg/tsdb/elasticsearch/client" @@ -12,15 +13,20 @@ import ( // ElasticsearchExecutor represents a handler for handling elasticsearch datasource request type Executor struct { + httpClientProvider httpclient.Provider intervalCalculator interval.Calculator } -// NewExecutor creates a new Executor. -//nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) { - return &Executor{ - intervalCalculator: interval.NewCalculator(), - }, nil +// New creates a new Executor func. +// nolint:staticcheck // plugins.DataPlugin deprecated +func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + // nolint:staticcheck // plugins.DataPlugin deprecated + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + return &Executor{ + httpClientProvider: httpClientProvider, + intervalCalculator: interval.NewCalculator(), + }, nil + } } // Query handles an elasticsearch datasource request @@ -31,8 +37,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, return plugins.DataResponse{}, fmt.Errorf("query contains no queries") } - client, err := es.NewClient(ctx, dsInfo, *tsdbQuery.TimeRange) - + client, err := es.NewClient(ctx, e.httpClientProvider, dsInfo, *tsdbQuery.TimeRange) if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/graphite/graphite.go b/pkg/tsdb/graphite/graphite.go index 9f24fbf73bd..4db5c966c29 100644 --- a/pkg/tsdb/graphite/graphite.go +++ b/pkg/tsdb/graphite/graphite.go @@ -15,6 +15,7 @@ import ( "golang.org/x/net/context/ctxhttp" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -23,12 +24,17 @@ import ( ) type GraphiteExecutor struct { - HttpClient *http.Client + httpClientProvider httpclient.Provider } -//nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) { - return &GraphiteExecutor{}, nil +// nolint:staticcheck // plugins.DataPlugin deprecated +func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + // nolint:staticcheck // plugins.DataPlugin deprecated + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + return &GraphiteExecutor{ + httpClientProvider: httpClientProvider, + }, nil + } } var glog = log.New("tsdb.graphite") @@ -91,7 +97,7 @@ func (e *GraphiteExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou return plugins.DataResponse{}, err } - httpClient, err := dsInfo.GetHttpClient() + httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider) if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/influxdb/flux/executor_test.go b/pkg/tsdb/influxdb/flux/executor_test.go index 332e5dc6143..5b39cfabecf 100644 --- a/pkg/tsdb/influxdb/flux/executor_test.go +++ b/pkg/tsdb/influxdb/flux/executor_test.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental" "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -226,7 +227,7 @@ func TestRealQuery(t *testing.T) { }), } - runner, err := runnerFromDataSource(dsInfo) + runner, err := runnerFromDataSource(httpclient.NewProvider(), dsInfo) require.NoError(t, err) dr := executeQuery(context.Background(), queryModel{ diff --git a/pkg/tsdb/influxdb/flux/flux.go b/pkg/tsdb/influxdb/flux/flux.go index 1b5c53a23bd..24b6e3f961d 100644 --- a/pkg/tsdb/influxdb/flux/flux.go +++ b/pkg/tsdb/influxdb/flux/flux.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -22,13 +23,13 @@ func init() { // Query builds flux queries, executes them, and returns the results. //nolint: staticcheck // plugins.DataQuery deprecated -func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) ( +func Query(ctx context.Context, httpClientProvider httpclient.Provider, dsInfo *models.DataSource, tsdbQuery plugins.DataQuery) ( plugins.DataResponse, error) { glog.Debug("Received a query", "query", tsdbQuery) tRes := plugins.DataResponse{ Results: make(map[string]plugins.DataQueryResult), } - r, err := runnerFromDataSource(dsInfo) + r, err := runnerFromDataSource(httpClientProvider, dsInfo) if err != nil { return plugins.DataResponse{}, err } @@ -69,7 +70,7 @@ func (r *runner) runQuery(ctx context.Context, fluxQuery string) (*api.QueryTabl } // runnerFromDataSource creates a runner from the datasource model (the datasource instance's configuration). -func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) { +func runnerFromDataSource(httpClientProvider httpclient.Provider, dsInfo *models.DataSource) (*runner, error) { org := dsInfo.JsonData.Get("organization").MustString("") if org == "" { return nil, fmt.Errorf("missing organization in datasource configuration") @@ -85,7 +86,7 @@ func runnerFromDataSource(dsInfo *models.DataSource) (*runner, error) { } opts := influxdb2.DefaultOptions() - hc, err := dsInfo.GetHttpClient() + hc, err := dsInfo.GetHTTPClient(httpClientProvider) if err != nil { return nil, err } diff --git a/pkg/tsdb/influxdb/influxdb.go b/pkg/tsdb/influxdb/influxdb.go index 94298d9e377..c42b868a161 100644 --- a/pkg/tsdb/influxdb/influxdb.go +++ b/pkg/tsdb/influxdb/influxdb.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -17,17 +18,21 @@ import ( ) type Executor struct { - // *models.DataSource - QueryParser *InfluxdbQueryParser - ResponseParser *ResponseParser + httpClientProvider httpclient.Provider + QueryParser *InfluxdbQueryParser + ResponseParser *ResponseParser } -//nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) { - return &Executor{ - QueryParser: &InfluxdbQueryParser{}, - ResponseParser: &ResponseParser{}, - }, nil +// nolint:staticcheck // plugins.DataPlugin deprecated +func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + // nolint:staticcheck // plugins.DataPlugin deprecated + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + return &Executor{ + httpClientProvider: httpClientProvider, + QueryParser: &InfluxdbQueryParser{}, + ResponseParser: &ResponseParser{}, + }, nil + } } var ( @@ -47,7 +52,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd version := dsInfo.JsonData.Get("version").MustString("") if version == "Flux" { - return flux.Query(ctx, dsInfo, tsdbQuery) + return flux.Query(ctx, e.httpClientProvider, dsInfo, tsdbQuery) } glog.Debug("Making a non-Flux type query") @@ -74,7 +79,7 @@ func (e *Executor) DataQuery(ctx context.Context, dsInfo *models.DataSource, tsd return plugins.DataResponse{}, err } - httpClient, err := dsInfo.GetHttpClient() + httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider) if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index 946380f028d..f0a3c04a94e 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -9,6 +9,7 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -23,16 +24,17 @@ import ( type LokiExecutor struct { intervalCalculator interval.Calculator + httpClientProvider httpclient.Provider } -//nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) { - return newExecutor(), nil -} - -func newExecutor() *LokiExecutor { - return &LokiExecutor{ - intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}), +// nolint:staticcheck // plugins.DataPlugin deprecated +func New(httpClientProvider httpclient.Provider) func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + // nolint:staticcheck // plugins.DataPlugin deprecated + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + return &LokiExecutor{ + intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}), + httpClientProvider: httpClientProvider, + }, nil } } @@ -49,12 +51,12 @@ func (e *LokiExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSource, Results: map[string]plugins.DataQueryResult{}, } - tlsConfig, err := dsInfo.GetTLSConfig() + tlsConfig, err := dsInfo.GetTLSConfig(e.httpClientProvider) if err != nil { return plugins.DataResponse{}, err } - transport, err := dsInfo.GetHttpTransport() + transport, err := dsInfo.GetHTTPTransport(e.httpClientProvider) if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/loki/loki_test.go b/pkg/tsdb/loki/loki_test.go index f7d4dd4683e..df3be701fa3 100644 --- a/pkg/tsdb/loki/loki_test.go +++ b/pkg/tsdb/loki/loki_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/loki/pkg/loghttp" @@ -60,9 +61,10 @@ func TestLoki(t *testing.T) { TimeRange: &timeRange, } - exe := newExecutor() + exe, err := New(httpclient.NewProvider())(dsInfo) require.NoError(t, err) - models, err := exe.parseQuery(dsInfo, queryContext) + lokiExecutor := exe.(*LokiExecutor) + models, err := lokiExecutor.parseQuery(dsInfo, queryContext) require.NoError(t, err) require.Equal(t, time.Second*30, models[0].Step) }) @@ -82,15 +84,16 @@ func TestLoki(t *testing.T) { {Model: jsonModel}, }, } - exe := newExecutor() + exe, err := New(httpclient.NewProvider())(dsInfo) require.NoError(t, err) - models, err := exe.parseQuery(dsInfo, queryContext) + lokiExecutor := exe.(*LokiExecutor) + models, err := lokiExecutor.parseQuery(dsInfo, queryContext) require.NoError(t, err) require.Equal(t, time.Minute*2, models[0].Step) timeRange = plugins.NewDataTimeRange("1h", "now") queryContext.TimeRange = &timeRange - models, err = exe.parseQuery(dsInfo, queryContext) + models, err = lokiExecutor.parseQuery(dsInfo, queryContext) require.NoError(t, err) require.Equal(t, time.Second*2, models[0].Step) }) diff --git a/pkg/tsdb/mysql/mysql.go b/pkg/tsdb/mysql/mysql.go index 07fc15f2c05..1ff3b23089c 100644 --- a/pkg/tsdb/mysql/mysql.go +++ b/pkg/tsdb/mysql/mysql.go @@ -12,6 +12,7 @@ import ( "github.com/VividCortex/mysqlerr" "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data/sqlutil" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/setting" "github.com/go-sql-driver/mysql" @@ -32,52 +33,55 @@ func characterEscape(s string, escapeChar string) string { } //nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(datasource *models.DataSource) (plugins.DataPlugin, error) { - logger := log.New("tsdb.mysql") +func New(httpClientProvider httpclient.Provider) func(datasource *models.DataSource) (plugins.DataPlugin, error) { + //nolint: staticcheck // plugins.DataPlugin deprecated + return func(datasource *models.DataSource) (plugins.DataPlugin, error) { + logger := log.New("tsdb.mysql") - protocol := "tcp" - if strings.HasPrefix(datasource.Url, "/") { - protocol = "unix" - } - - cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true", - characterEscape(datasource.User, ":"), - datasource.DecryptedPassword(), - protocol, - characterEscape(datasource.Url, ")"), - characterEscape(datasource.Database, "?"), - ) + protocol := "tcp" + if strings.HasPrefix(datasource.Url, "/") { + protocol = "unix" + } - tlsConfig, err := datasource.GetTLSConfig() - if err != nil { - return nil, err - } + cnnstr := fmt.Sprintf("%s:%s@%s(%s)/%s?collation=utf8mb4_unicode_ci&parseTime=true&loc=UTC&allowNativePasswords=true", + characterEscape(datasource.User, ":"), + datasource.DecryptedPassword(), + protocol, + characterEscape(datasource.Url, ")"), + characterEscape(datasource.Database, "?"), + ) - if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 { - tlsConfigString := fmt.Sprintf("ds%d", datasource.Id) - if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil { + tlsConfig, err := datasource.GetTLSConfig(httpClientProvider) + if err != nil { return nil, err } - cnnstr += "&tls=" + tlsConfigString - } - if setting.Env == setting.Dev { - logger.Debug("getEngine", "connection", cnnstr) - } + if tlsConfig.RootCAs != nil || len(tlsConfig.Certificates) > 0 { + tlsConfigString := fmt.Sprintf("ds%d", datasource.Id) + if err := mysql.RegisterTLSConfig(tlsConfigString, tlsConfig); err != nil { + return nil, err + } + cnnstr += "&tls=" + tlsConfigString + } - config := sqleng.DataPluginConfiguration{ - DriverName: "mysql", - ConnectionString: cnnstr, - Datasource: datasource, - TimeColumnNames: []string{"time", "time_sec"}, - MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"}, - } + if setting.Env == setting.Dev { + logger.Debug("getEngine", "connection", cnnstr) + } - rowTransformer := mysqlQueryResultTransformer{ - log: logger, - } + config := sqleng.DataPluginConfiguration{ + DriverName: "mysql", + ConnectionString: cnnstr, + Datasource: datasource, + TimeColumnNames: []string{"time", "time_sec"}, + MetricColumnTypes: []string{"CHAR", "VARCHAR", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"}, + } - return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger) + rowTransformer := mysqlQueryResultTransformer{ + log: logger, + } + + return sqleng.NewDataPlugin(config, &rowTransformer, newMysqlMacroEngine(logger), logger) + } } type mysqlQueryResultTransformer struct { diff --git a/pkg/tsdb/mysql/mysql_test.go b/pkg/tsdb/mysql/mysql_test.go index cca34b62b4c..b0025ae14e7 100644 --- a/pkg/tsdb/mysql/mysql_test.go +++ b/pkg/tsdb/mysql/mysql_test.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/components/securejsondata" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/sqlstore" @@ -56,7 +57,7 @@ func TestMySQL(t *testing.T) { return sql, nil } - exe, err := NewExecutor(&models.DataSource{ + exe, err := New(httpclient.NewProvider())(&models.DataSource{ JsonData: simplejson.New(), SecureJsonData: securejsondata.SecureJsonData{}, }) diff --git a/pkg/tsdb/opentsdb/opentsdb.go b/pkg/tsdb/opentsdb/opentsdb.go index 4c94cf7a321..859ac967efb 100644 --- a/pkg/tsdb/opentsdb/opentsdb.go +++ b/pkg/tsdb/opentsdb/opentsdb.go @@ -15,6 +15,7 @@ import ( "net/url" "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -22,11 +23,17 @@ import ( ) type OpenTsdbExecutor struct { + httpClientProvider httpclient.Provider } -// nolint:staticcheck // plugins.DataQueryResult deprecated -func NewExecutor(*models.DataSource) (plugins.DataPlugin, error) { - return &OpenTsdbExecutor{}, nil +//nolint: staticcheck // plugins.DataPlugin deprecated +func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + //nolint: staticcheck // plugins.DataPlugin deprecated + return func(*models.DataSource) (plugins.DataPlugin, error) { + return &OpenTsdbExecutor{ + httpClientProvider: httpClientProvider, + }, nil + } } var ( @@ -56,7 +63,7 @@ func (e *OpenTsdbExecutor) DataQuery(ctx context.Context, dsInfo *models.DataSou return plugins.DataResponse{}, err } - httpClient, err := dsInfo.GetHttpClient() + httpClient, err := dsInfo.GetHTTPClient(e.httpClientProvider) if err != nil { return plugins.DataResponse{}, err } diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 4546deb320c..91c7e869010 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -14,6 +14,7 @@ import ( "net/http" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -65,13 +66,20 @@ func (transport *prometheusTransport) RoundTrip(req *http.Request) (*http.Respon } //nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) { - return &PrometheusExecutor{ - intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}), - baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) { - return ds.GetHttpTransport() - }, - }, nil +func New(provider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + transport, err := dsInfo.GetHTTPTransport(provider) + if err != nil { + return nil, err + } + + return &PrometheusExecutor{ + intervalCalculator: interval.NewCalculator(interval.CalculatorOptions{MinInterval: time.Second * 1}), + baseRoundTripperFactory: func(ds *models.DataSource) (http.RoundTripper, error) { + return transport, nil + }, + }, nil + } } var ( diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go index 86598765390..6fead701028 100644 --- a/pkg/tsdb/prometheus/prometheus_test.go +++ b/pkg/tsdb/prometheus/prometheus_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" p "github.com/prometheus/common/model" @@ -21,7 +22,7 @@ func TestPrometheus(t *testing.T) { dsInfo := &models.DataSource{ JsonData: json, } - plug, err := NewExecutor(dsInfo) + plug, err := New(httpclient.NewProvider())(dsInfo) require.NoError(t, err) executor := plug.(*PrometheusExecutor) diff --git a/pkg/tsdb/service.go b/pkg/tsdb/service.go index ae1d90beea3..aeccab78e56 100644 --- a/pkg/tsdb/service.go +++ b/pkg/tsdb/service.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry" @@ -47,6 +48,7 @@ type Service struct { CloudMonitoringService *cloudmonitoring.Service `inject:""` AzureMonitorService *azuremonitor.Service `inject:""` PluginManager plugins.Manager `inject:""` + HTTPClientProvider httpclient.Provider `inject:""` //nolint: staticcheck // plugins.DataPlugin deprecated registry map[string]func(*models.DataSource) (plugins.DataPlugin, error) @@ -54,18 +56,18 @@ type Service struct { // Init initialises the service. func (s *Service) Init() error { - s.registry["graphite"] = graphite.NewExecutor - s.registry["opentsdb"] = opentsdb.NewExecutor - s.registry["prometheus"] = prometheus.NewExecutor - s.registry["influxdb"] = influxdb.NewExecutor + s.registry["graphite"] = graphite.New(s.HTTPClientProvider) + s.registry["opentsdb"] = opentsdb.New(s.HTTPClientProvider) + s.registry["prometheus"] = prometheus.New(s.HTTPClientProvider) + s.registry["influxdb"] = influxdb.New(s.HTTPClientProvider) s.registry["mssql"] = mssql.NewExecutor s.registry["postgres"] = s.PostgresService.NewExecutor - s.registry["mysql"] = mysql.NewExecutor - s.registry["elasticsearch"] = elasticsearch.NewExecutor + s.registry["mysql"] = mysql.New(s.HTTPClientProvider) + s.registry["elasticsearch"] = elasticsearch.New(s.HTTPClientProvider) s.registry["stackdriver"] = s.CloudMonitoringService.NewExecutor s.registry["grafana-azure-monitor-datasource"] = s.AzureMonitorService.NewExecutor - s.registry["loki"] = loki.NewExecutor - s.registry["tempo"] = tempo.NewExecutor + s.registry["loki"] = loki.New(s.HTTPClientProvider) + s.registry["tempo"] = tempo.New(s.HTTPClientProvider) return nil } diff --git a/pkg/tsdb/tempo/tempo.go b/pkg/tsdb/tempo/tempo.go index b6444dcf8cf..8ce7493606f 100644 --- a/pkg/tsdb/tempo/tempo.go +++ b/pkg/tsdb/tempo/tempo.go @@ -7,6 +7,7 @@ import ( "net/http" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" @@ -18,17 +19,20 @@ type tempoExecutor struct { httpClient *http.Client } -// NewExecutor returns a tempoExecutor.DataQueryResult +// NewExecutor returns a tempoExecutor. //nolint: staticcheck // plugins.DataPlugin deprecated -func NewExecutor(dsInfo *models.DataSource) (plugins.DataPlugin, error) { - httpClient, err := dsInfo.GetHttpClient() - if err != nil { - return nil, err - } +func New(httpClientProvider httpclient.Provider) func(*models.DataSource) (plugins.DataPlugin, error) { + //nolint: staticcheck // plugins.DataPlugin deprecated + return func(dsInfo *models.DataSource) (plugins.DataPlugin, error) { + httpClient, err := dsInfo.GetHTTPClient(httpClientProvider) + if err != nil { + return nil, err + } - return &tempoExecutor{ - httpClient: httpClient, - }, nil + return &tempoExecutor{ + httpClient: httpClient, + }, nil + } } var ( diff --git a/pkg/tsdb/tempo/tempo_test.go b/pkg/tsdb/tempo/tempo_test.go index 14dce746bf5..8dcb0ea757c 100644 --- a/pkg/tsdb/tempo/tempo_test.go +++ b/pkg/tsdb/tempo/tempo_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" + "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestTempo(t *testing.T) { - plug, err := NewExecutor(&models.DataSource{}) + plug, err := New(httpclient.NewProvider())(&models.DataSource{}) executor := plug.(*tempoExecutor) require.NoError(t, err)