Datasource: Add support for max_conns_per_host in dataproxy settings (#35520)

Allow configuring dataproxy.max_conns_per_host for HTTP data sources.

Ref #35519
Ref #35257

conf/defaults.ini

@@ -158,6 +158,11 @@ tls_handshake_timeout_seconds = 10
 # waiting for the server to approve.
 expect_continue_timeout_seconds = 1
 
+# Optionally limits the total number of connections per host, including connections in the dialing,
+# active, and idle states. On limit violation, dials will block.
+# A value of zero (0) means no limit.
+max_conns_per_host = 0
+
 # The maximum number of idle connections that Grafana will keep alive.
 max_idle_connections = 100
 

conf/sample.ini

@@ -164,6 +164,11 @@
 # waiting for the server to approve.
 ;expect_continue_timeout_seconds = 1
 
+# Optionally limits the total number of connections per host, including connections in the dialing,
+# active, and idle states. On limit violation, dials will block.
+# A value of zero (0) means no limit.
+;max_conns_per_host = 0
+
 # The maximum number of idle connections that Grafana will keep alive.
 ;max_idle_connections = 100
 

docs/sources/administration/configuration.md

@@ -414,6 +414,11 @@ The length of time that Grafana will wait for a successful TLS handshake with th
 
 The length of time that Grafana will wait for a datasource’s first response headers after fully writing the request headers, if the request has an “Expect: 100-continue” header. A value of `0` will result in the body being sent immediately. Default is `1` second. For more details check the [Transport.ExpectContinueTimeout](https://golang.org/pkg/net/http/#Transport.ExpectContinueTimeout) documentation.
 
+### max_conns_per_host
+
+Optionally limits the total number of connections per host, including connections in the dialing, active, and idle states. On limit violation, dials are blocked. A value of `0` means that there are no limits. Default is `0`.
+For more details check the [Transport.MaxConnsPerHost](https://golang.org/pkg/net/http/#Transport.MaxConnsPerHost) documentation.
+
 ### max_idle_connections
 
 The maximum number of idle connections that Grafana will maintain. Default is `100`. For more details check the [Transport.MaxIdleConns](https://golang.org/pkg/net/http/#Transport.MaxIdleConns) documentation.

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/gosimple/slug v1.9.0
 	github.com/grafana/grafana-aws-sdk v0.4.0
-	github.com/grafana/grafana-plugin-sdk-go v0.104.0
+	github.com/grafana/grafana-plugin-sdk-go v0.105.0
 	github.com/grafana/loki v1.6.2-0.20210520072447-15d417efe103
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/hashicorp/go-hclog v0.16.0

go.sum

@@ -915,8 +915,8 @@ github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0r
 github.com/grafana/grafana-aws-sdk v0.4.0 h1:JmTaXfOJ/ydHSWH9kEt8Yhfb9kAhIW4LUOO3SWCviYg=
 github.com/grafana/grafana-aws-sdk v0.4.0/go.mod h1:+pPo5U+pX0zWimR7YBc7ASeSQfbRkcTyQYqMiAj7G5U=
 github.com/grafana/grafana-plugin-sdk-go v0.79.0/go.mod h1:NvxLzGkVhnoBKwzkst6CFfpMFKwAdIUZ1q8ssuLeF60=
-github.com/grafana/grafana-plugin-sdk-go v0.104.0 h1:Ij2tPdEasSjCb2MxHaaiylyW4RLVZYyWpApzN/mlTxo=
-github.com/grafana/grafana-plugin-sdk-go v0.104.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk=
+github.com/grafana/grafana-plugin-sdk-go v0.105.0 h1:I0r88FtnXkWw4F0t36cmRCupizY4cPkK+6PKKqbyx9Q=
+github.com/grafana/grafana-plugin-sdk-go v0.105.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk=
 github.com/grafana/loki v1.6.2-0.20210520072447-15d417efe103 h1:qCmofFVwQR9QnsinstVqI1NPLMVl33jNCnOCXEAVn6E=
 github.com/grafana/loki v1.6.2-0.20210520072447-15d417efe103/go.mod h1:GHIsn+EohCChsdu5YouNZewqLeV9L2FNw4DEJU3P9qE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=

pkg/models/datasource_cache.go

@@ -91,6 +91,7 @@ func (ds *DataSource) HTTPClientOptions() sdkhttpclient.Options {
 			KeepAlive:             time.Duration(setting.DataProxyKeepAlive) * time.Second,
 			TLSHandshakeTimeout:   time.Duration(setting.DataProxyTLSHandshakeTimeout) * time.Second,
 			ExpectContinueTimeout: time.Duration(setting.DataProxyExpectContinueTimeout) * time.Second,
+			MaxConnsPerHost:       setting.DataProxyMaxConnsPerHost,
 			MaxIdleConns:          setting.DataProxyMaxIdleConns,
 			MaxIdleConnsPerHost:   setting.DataProxyMaxIdleConnsPerHost,
 			IdleConnTimeout:       time.Duration(setting.DataProxyIdleConnTimeout) * time.Second,

pkg/setting/setting.go

@@ -81,6 +81,7 @@ var (
 	DataProxyDialTimeout           int
 	DataProxyTLSHandshakeTimeout   int
 	DataProxyExpectContinueTimeout int
+	DataProxyMaxConnsPerHost       int
 	DataProxyMaxIdleConns          int
 	DataProxyMaxIdleConnsPerHost   int
 	DataProxyKeepAlive             int
@@ -838,6 +839,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	DataProxyKeepAlive = dataproxy.Key("keep_alive_seconds").MustInt(30)
 	DataProxyTLSHandshakeTimeout = dataproxy.Key("tls_handshake_timeout_seconds").MustInt(10)
 	DataProxyExpectContinueTimeout = dataproxy.Key("expect_continue_timeout_seconds").MustInt(1)
+	DataProxyMaxConnsPerHost = dataproxy.Key("max_conns_per_host").MustInt(0)
 	DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt(100)
 	DataProxyMaxIdleConnsPerHost = dataproxy.Key("max_idle_connections_per_host").MustInt(2)
 	DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)