mirror of https://github.com/grafana/grafana
Use cache for http.client in tsdb package. (#6833)
* datasource: move caching closer to datasource struct * tsdb: use cached version of datasource http transport closes #6825pull/2317/merge
Carl Bergquist
9 years ago
committed by
Torkel Ödegaard
parent
24172fca01
commit
2c7adccf12
@ -0,0 +1,95 @@ |
||||
package models |
||||
|
||||
import ( |
||||
"crypto/tls" |
||||
"crypto/x509" |
||||
"net" |
||||
"net/http" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
type proxyTransportCache struct { |
||||
cache map[int64]cachedTransport |
||||
sync.Mutex |
||||
} |
||||
|
||||
type cachedTransport struct { |
||||
updated time.Time |
||||
|
||||
*http.Transport |
||||
} |
||||
|
||||
var ptc = proxyTransportCache{ |
||||
cache: make(map[int64]cachedTransport), |
||||
} |
||||
|
||||
func (ds *DataSource) GetHttpClient() (*http.Client, error) { |
||||
transport, err := ds.GetHttpTransport() |
||||
|
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return &http.Client{ |
||||
Timeout: time.Duration(30 * time.Second), |
||||
Transport: transport, |
||||
}, nil |
||||
} |
||||
|
||||
func (ds *DataSource) GetHttpTransport() (*http.Transport, error) { |
||||
ptc.Lock() |
||||
defer ptc.Unlock() |
||||
|
||||
if t, present := ptc.cache[ds.Id]; present && ds.Updated.Equal(t.updated) { |
||||
return t.Transport, nil |
||||
} |
||||
|
||||
transport := &http.Transport{ |
||||
TLSClientConfig: &tls.Config{ |
||||
InsecureSkipVerify: true, |
||||
}, |
||||
Proxy: http.ProxyFromEnvironment, |
||||
Dial: (&net.Dialer{ |
||||
Timeout: 30 * time.Second, |
||||
KeepAlive: 30 * time.Second, |
||||
}).Dial, |
||||
TLSHandshakeTimeout: 10 * time.Second, |
||||
ExpectContinueTimeout: 1 * time.Second, |
||||
MaxIdleConns: 100, |
||||
IdleConnTimeout: 90 * time.Second, |
||||
} |
||||
|
||||
var tlsAuth, tlsAuthWithCACert bool |
||||
if ds.JsonData != nil { |
||||
tlsAuth = ds.JsonData.Get("tlsAuth").MustBool(false) |
||||
tlsAuthWithCACert = ds.JsonData.Get("tlsAuthWithCACert").MustBool(false) |
||||
} |
||||
|
||||
if tlsAuth { |
||||
transport.TLSClientConfig.InsecureSkipVerify = false |
||||
|
||||
decrypted := ds.SecureJsonData.Decrypt() |
||||
|
||||
if tlsAuthWithCACert && len(decrypted["tlsCACert"]) > 0 { |
||||
caPool := x509.NewCertPool() |
||||
ok := caPool.AppendCertsFromPEM([]byte(decrypted["tlsCACert"])) |
||||
if ok { |
||||
transport.TLSClientConfig.RootCAs = caPool |
||||
} |
||||
} |
||||
|
||||
cert, err := tls.X509KeyPair([]byte(decrypted["tlsClientCert"]), []byte(decrypted["tlsClientKey"])) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
transport.TLSClientConfig.Certificates = []tls.Certificate{cert} |
||||
} |
||||
|
||||
ptc.cache[ds.Id] = cachedTransport{ |
||||
Transport: transport, |
||||
updated: ds.Updated, |
||||
} |
||||
|
||||
return transport, nil |
||||
} |
||||
@ -0,0 +1,157 @@ |
||||
package models |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
. "github.com/smartystreets/goconvey/convey" |
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson" |
||||
"github.com/grafana/grafana/pkg/setting" |
||||
"github.com/grafana/grafana/pkg/util" |
||||
) |
||||
|
||||
func TestDataSourceCache(t *testing.T) { |
||||
Convey("When caching a datasource proxy", t, func() { |
||||
clearCache() |
||||
ds := DataSource{ |
||||
Id: 1, |
||||
Url: "http://k8s:8001", |
||||
Type: "Kubernetes", |
||||
} |
||||
|
||||
t1, err := ds.GetHttpTransport() |
||||
So(err, ShouldBeNil) |
||||
|
||||
t2, err := ds.GetHttpTransport() |
||||
So(err, ShouldBeNil) |
||||
|
||||
Convey("Should be using the cached proxy", func() { |
||||
So(t2, ShouldEqual, t1) |
||||
}) |
||||
}) |
||||
|
||||
Convey("When getting kubernetes datasource proxy", t, func() { |
||||
clearCache() |
||||
setting.SecretKey = "password" |
||||
|
||||
json := simplejson.New() |
||||
json.Set("tlsAuth", true) |
||||
json.Set("tlsAuthWithCACert", true) |
||||
|
||||
t := time.Now() |
||||
ds := DataSource{ |
||||
Url: "http://k8s:8001", |
||||
Type: "Kubernetes", |
||||
Updated: t.Add(-2 * time.Minute), |
||||
} |
||||
|
||||
transport, err := ds.GetHttpTransport() |
||||
So(err, ShouldBeNil) |
||||
|
||||
Convey("Should have no cert", func() { |
||||
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, true) |
||||
}) |
||||
|
||||
ds.JsonData = json |
||||
ds.SecureJsonData = map[string][]byte{ |
||||
"tlsCACert": util.Encrypt([]byte(caCert), "password"), |
||||
"tlsClientCert": util.Encrypt([]byte(clientCert), "password"), |
||||
"tlsClientKey": util.Encrypt([]byte(clientKey), "password"), |
||||
} |
||||
ds.Updated = t.Add(-1 * time.Minute) |
||||
|
||||
transport, err = ds.GetHttpTransport() |
||||
So(err, ShouldBeNil) |
||||
|
||||
Convey("Should add cert", func() { |
||||
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, false) |
||||
So(len(transport.TLSClientConfig.Certificates), ShouldEqual, 1) |
||||
}) |
||||
|
||||
ds.JsonData = nil |
||||
ds.SecureJsonData = map[string][]byte{} |
||||
ds.Updated = t |
||||
|
||||
transport, err = ds.GetHttpTransport() |
||||
So(err, ShouldBeNil) |
||||
|
||||
Convey("Should remove cert", func() { |
||||
So(transport.TLSClientConfig.InsecureSkipVerify, ShouldEqual, true) |
||||
So(len(transport.TLSClientConfig.Certificates), ShouldEqual, 0) |
||||
}) |
||||
}) |
||||
} |
||||
|
||||
func clearCache() { |
||||
ptc.Lock() |
||||
defer ptc.Unlock() |
||||
|
||||
ptc.cache = make(map[int64]cachedTransport) |
||||
} |
||||
|
||||
const caCert string = `-----BEGIN CERTIFICATE----- |
||||
MIIDATCCAemgAwIBAgIJAMQ5hC3CPDTeMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV |
||||
BAMMDGNhLWs4cy1zdGhsbTAeFw0xNjEwMjcwODQyMjdaFw00NDAzMTQwODQyMjda |
||||
MBcxFTATBgNVBAMMDGNhLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQADggEP |
||||
ADCCAQoCggEBAMLe2AmJ6IleeUt69vgNchOjjmxIIxz5sp1vFu94m1vUip7CqnOg |
||||
QkpUsHeBPrGYv8UGloARCL1xEWS+9FVZeXWQoDmbC0SxXhFwRIESNCET7Q8KMi/4 |
||||
4YPvnMLGZi3Fjwxa8BdUBCN1cx4WEooMVTWXm7RFMtZgDfuOAn3TNXla732sfT/d |
||||
1HNFrh48b0wA+HhmA3nXoBnBEblA665hCeo7lIAdRr0zJxJpnFnWXkyTClsAUTMN |
||||
iL905LdBiiIRenojipfKXvMz88XSaWTI7JjZYU3BvhyXndkT6f12cef3I96NY3WJ |
||||
0uIK4k04WrbzdYXMU3rN6NqlvbHqnI+E7aMCAwEAAaNQME4wHQYDVR0OBBYEFHHx |
||||
2+vSPw9bECHj3O51KNo5VdWOMB8GA1UdIwQYMBaAFHHx2+vSPw9bECHj3O51KNo5 |
||||
VdWOMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAH2eV5NcV3LBJHs9 |
||||
I+adbiTPg2vyumrGWwy73T0X8Dtchgt8wU7Q9b9Ucg2fOTmSSyS0iMqEu1Yb2ORB |
||||
CknM9mixHC9PwEBbkGCom3VVkqdLwSP6gdILZgyLoH4i8sTUz+S1yGPepi+Vzhs7 |
||||
adOXtryjcGnwft6HdfKPNklMOHFnjw6uqpho54oj/z55jUpicY/8glDHdrr1bh3k |
||||
MHuiWLGewHXPvxfG6UoUx1te65IhifVcJGFZDQwfEmhBflfCmtAJlZEsgTLlBBCh |
||||
FHoXIyGOdq1chmRVocdGBCF8fUoGIbuF14r53rpvcbEKtKnnP8+96luKAZLq0a4n |
||||
3lb92xM= |
||||
-----END CERTIFICATE-----` |
||||
|
||||
const clientCert string = `-----BEGIN CERTIFICATE----- |
||||
MIICsjCCAZoCCQCcd8sOfstQLzANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQDDAxj |
||||
YS1rOHMtc3RobG0wHhcNMTYxMTAyMDkyNTE1WhcNMTcxMTAyMDkyNTE1WjAfMR0w |
||||
GwYDVQQDDBRhZG0tZGFuaWVsLWs4cy1zdGhsbTCCASIwDQYJKoZIhvcNAQEBBQAD |
||||
ggEPADCCAQoCggEBAOMliaWyNEUJKM37vWCl5bGub3lMicyRAqGQyY/qxD9yKKM2 |
||||
FbucVcmWmg5vvTqQVl5rlQ+c7GI8OD6ptmFl8a26coEki7bFr8bkpSyBSEc5p27b |
||||
Z0ORFSqBHWHQbr9PkxPLYW6T3gZYUtRYv3OQgGxLXlvUh85n/mQfuR3N1FgmShHo |
||||
GtAFi/ht6leXa0Ms+jNSDLCmXpJm1GIEqgyKX7K3+g3vzo9coYqXq4XTa8Efs2v8 |
||||
SCwqWfBC3rHfgs/5DLB8WT4Kul8QzxkytzcaBQfRfzhSV6bkgm7oTzt2/1eRRsf4 |
||||
YnXzLE9YkCC9sAn+Owzqf+TYC1KRluWDfqqBTJUCAwEAATANBgkqhkiG9w0BAQsF |
||||
AAOCAQEAdMsZg6edWGC+xngizn0uamrUg1ViaDqUsz0vpzY5NWLA4MsBc4EtxWRP |
||||
ueQvjUimZ3U3+AX0YWNLIrH1FCVos2jdij/xkTUmHcwzr8rQy+B17cFi+a8jtpgw |
||||
AU6WWoaAIEhhbWQfth/Diz3mivl1ARB+YqiWca2mjRPLTPcKJEURDVddQ423el0Q |
||||
4JNxS5icu7T2zYTYHAo/cT9zVdLZl0xuLxYm3asK1IONJ/evxyVZima3il6MPvhe |
||||
58Hwz+m+HdqHxi24b/1J/VKYbISG4huOQCdLzeNXgvwFlGPUmHSnnKo1/KbQDAR5 |
||||
llG/Sw5+FquFuChaA6l5KWy7F3bQyA== |
||||
-----END CERTIFICATE-----` |
||||
|
||||
const clientKey string = `-----BEGIN RSA PRIVATE KEY----- |
||||
MIIEpQIBAAKCAQEA4yWJpbI0RQkozfu9YKXlsa5veUyJzJECoZDJj+rEP3IoozYV |
||||
u5xVyZaaDm+9OpBWXmuVD5zsYjw4Pqm2YWXxrbpygSSLtsWvxuSlLIFIRzmnbttn |
||||
Q5EVKoEdYdBuv0+TE8thbpPeBlhS1Fi/c5CAbEteW9SHzmf+ZB+5Hc3UWCZKEega |
||||
0AWL+G3qV5drQyz6M1IMsKZekmbUYgSqDIpfsrf6De/Oj1yhiperhdNrwR+za/xI |
||||
LCpZ8ELesd+Cz/kMsHxZPgq6XxDPGTK3NxoFB9F/OFJXpuSCbuhPO3b/V5FGx/hi |
||||
dfMsT1iQIL2wCf47DOp/5NgLUpGW5YN+qoFMlQIDAQABAoIBAQCzy4u312XeW1Cs |
||||
Mx6EuOwmh59/ESFmBkZh4rxZKYgrfE5EWlQ7i5SwG4BX+wR6rbNfy6JSmHDXlTkk |
||||
CKvvToVNcW6fYHEivDnVojhIERFIJ4+rhQmpBtcNLOQ3/4cZ8X/GxE6b+3lb5l+x |
||||
64mnjPLKRaIr5/+TVuebEy0xNTJmjnJ7yiB2HRz7uXEQaVSk/P7KAkkyl/9J3/LM |
||||
8N9AX1w6qDaNQZ4/P0++1H4SQenosM/b/GqGTomarEk/GE0NcB9rzmR9VCXa7FRh |
||||
WV5jyt9vUrwIEiK/6nUnOkGO8Ei3kB7Y+e+2m6WdaNoU5RAfqXmXa0Q/a0lLRruf |
||||
vTMo2WrBAoGBAPRaK4cx76Q+3SJ/wfznaPsMM06OSR8A3ctKdV+ip/lyKtb1W8Pz |
||||
k8MYQDH7GwPtSu5QD8doL00pPjugZL/ba7X9nAsI+pinyEErfnB9y7ORNEjIYYzs |
||||
DiqDKup7ANgw1gZvznWvb9Ge0WUSXvWS0pFkgootQAf+RmnnbWGH6l6RAoGBAO35 |
||||
aGUrLro5u9RD24uSXNU3NmojINIQFK5dHAT3yl0BBYstL43AEsye9lX95uMPTvOQ |
||||
Cqcn42Hjp/bSe3n0ObyOZeXVrWcDFAfE0wwB1BkvL1lpgnFO9+VQORlH4w3Ppnpo |
||||
jcPkR2TFeDaAYtvckhxe/Bk3OnuFmnsQ3VzM75fFAoGBAI6PvS2XeNU+yA3EtA01 |
||||
hg5SQ+zlHswz2TMuMeSmJZJnhY78f5mHlwIQOAPxGQXlf/4iP9J7en1uPpzTK3S0 |
||||
M9duK4hUqMA/w5oiIhbHjf0qDnMYVbG+V1V+SZ+cPBXmCDihKreGr5qBKnHpkfV8 |
||||
v9WL6o1rcRw4wiQvnaV1gsvBAoGBALtzVTczr6gDKCAIn5wuWy+cQSGTsBunjRLX |
||||
xuVm5iEiV+KMYkPvAx/pKzMLP96lRVR3ptyKgAKwl7LFk3u50+zh4gQLr35QH2wL |
||||
Lw7rNc3srAhrItPsFzqrWX6/cGuFoKYVS239l/sZzRppQPXcpb7xVvTp2whHcir0 |
||||
Wtnpl+TdAoGAGqKqo2KU3JoY3IuTDUk1dsNAm8jd9EWDh+s1x4aG4N79mwcss5GD |
||||
FF8MbFPneK7xQd8L6HisKUDAUi2NOyynM81LAftPkvN6ZuUVeFDfCL4vCA0HUXLD |
||||
+VrOhtUZkNNJlLMiVRJuQKUOGlg8PpObqYbstQAf/0/yFJMRHG82Tcg= |
||||
-----END RSA PRIVATE KEY-----` |
||||
@ -1,29 +0,0 @@ |
||||
package tsdb |
||||
|
||||
import ( |
||||
"crypto/tls" |
||||
"net" |
||||
"net/http" |
||||
"time" |
||||
) |
||||
|
||||
func GetDefaultClient() *http.Client { |
||||
tr := &http.Transport{ |
||||
Proxy: http.ProxyFromEnvironment, |
||||
DialContext: (&net.Dialer{ |
||||
Timeout: 30 * time.Second, |
||||
KeepAlive: 30 * time.Second, |
||||
}).DialContext, |
||||
MaxIdleConns: 100, |
||||
IdleConnTimeout: 90 * time.Second, |
||||
TLSHandshakeTimeout: 10 * time.Second, |
||||
ExpectContinueTimeout: 1 * time.Second, |
||||
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
||||
} |
||||
|
||||
return &http.Client{ |
||||
Timeout: time.Duration(30 * time.Second), |
||||
Transport: tr, |
||||
} |
||||
} |
||||
Loading…
Reference in new issue