apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

cloudwatch: Consolidate client logic (#25555)

Browse Source 
* cloudwatch: Consolidate client logic

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
pull/26515/head^2
Patrik Karlström 5 years ago committed by GitHub
parent 80edbbe314
commit 43ef052d57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1098 additions and 581 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 146
      pkg/tsdb/cloudwatch/cloudwatch.go
  2. 14
      pkg/tsdb/cloudwatch/credentials.go
  3. 510
      pkg/tsdb/cloudwatch/log_actions_test.go
  4. 106
      pkg/tsdb/cloudwatch/logs_client_mock.go
  5. 97
      pkg/tsdb/cloudwatch/metric_find_query.go
  6. 501
      pkg/tsdb/cloudwatch/metric_find_query_test.go
  7. 2
      pkg/tsdb/cloudwatch/query_transformer_test.go
  8. 133
      pkg/tsdb/cloudwatch/test_utils.go
  9. 2
      pkg/tsdb/cloudwatch/time_series_query_test.go

146
pkg/tsdb/cloudwatch/cloudwatch.go
Unescape View File

@ -9,11 +9,17 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/log"
@ -45,75 +51,95 @@ var plog = log.New("tsdb.cloudwatch")
var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
func init() { func init() {
tsdb.RegisterTsdbQueryEndpoint("cloudwatch", newcloudWatchExecutor) tsdb.RegisterTsdbQueryEndpoint("cloudwatch", func(ds *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return newExecutor(), nil
})
} }
func newcloudWatchExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) { func newExecutor() *cloudWatchExecutor {
e := &cloudWatchExecutor{ return &cloudWatchExecutor{
DataSource: datasource, logsClientsByRegion: map[string]cloudwatchlogsiface.CloudWatchLogsAPI{},
} }
dsInfo := e.getDSInfo(defaultRegion)
defaultLogsClient, err := retrieveLogsClient(dsInfo)
if err != nil {
return nil, err
}
e.logsClientsByRegion = map[string]*cloudwatchlogs.CloudWatchLogs{
dsInfo.Region: defaultLogsClient,
defaultRegion: defaultLogsClient,
}
return e, nil
} }
// cloudWatchExecutor executes CloudWatch requests. // cloudWatchExecutor executes CloudWatch requests.
type cloudWatchExecutor struct { type cloudWatchExecutor struct {
*models.DataSource *models.DataSource
ec2Svc ec2iface.EC2API
rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
logsClientsByRegion map[string](*cloudwatchlogs.CloudWatchLogs) ec2Client ec2iface.EC2API
mux sync.Mutex rgtaClient resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
logsClientsByRegion map[string]cloudwatchlogsiface.CloudWatchLogsAPI
mtx sync.Mutex
} }
func (e *cloudWatchExecutor) getCWClient(region string) (*cloudwatch.CloudWatch, error) { func (e *cloudWatchExecutor) newSession(region string) (*session.Session, error) {
datasourceInfo := e.getDSInfo(region) dsInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(datasourceInfo) creds, err := getCredentials(dsInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
sess, err := newSession(cfg) cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
return newSession(cfg)
}
func (e *cloudWatchExecutor) getCWClient(region string) (cloudwatchiface.CloudWatchAPI, error) {
sess, err := e.newSession(region)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return newCWClient(sess), nil
}
client := cloudwatch.New(sess, cfg) func (e *cloudWatchExecutor) getCWLogsClient(region string) (cloudwatchlogsiface.CloudWatchLogsAPI, error) {
e.mtx.Lock()
defer e.mtx.Unlock()
client.Handlers.Send.PushFront(func(r *request.Request) { if logsClient, ok := e.logsClientsByRegion[region]; ok {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) return logsClient, nil
}) }
return client, nil sess, err := e.newSession(region)
} if err != nil {
return nil, err
}
func (e *cloudWatchExecutor) getCWLogsClient(region string) (*cloudwatchlogs.CloudWatchLogs, error) { logsClient := newCWLogsClient(sess)
e.mux.Lock() e.logsClientsByRegion[region] = logsClient
defer e.mux.Unlock()
if logsClient, ok := e.logsClientsByRegion[region]; ok {
return logsClient, nil return logsClient, nil
}
func (e *cloudWatchExecutor) getEC2Client(region string) (ec2iface.EC2API, error) {
if e.ec2Client != nil {
return e.ec2Client, nil
} }
dsInfo := e.getDSInfo(region) sess, err := e.newSession(region)
newLogsClient, err := retrieveLogsClient(dsInfo)
if err != nil { if err != nil {
return nil, err return nil, err
} }
e.ec2Client = newEC2Client(sess)
return e.ec2Client, nil
}
e.logsClientsByRegion[region] = newLogsClient func (e *cloudWatchExecutor) getRGTAClient(region string) (resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI,
error) {
if e.rgtaClient != nil {
return e.rgtaClient, nil
}
return newLogsClient, nil sess, err := e.newSession(region)
if err != nil {
return nil, err
}
e.rgtaClient = newRGTAClient(sess)
return e.rgtaClient, nil
} }
func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
@ -279,26 +305,44 @@ func (e *cloudWatchExecutor) getDSInfo(region string) *datasourceInfo {
} }
} }
func retrieveLogsClient(dsInfo *datasourceInfo) (*cloudwatchlogs.CloudWatchLogs, error) { func isTerminated(queryStatus string) bool {
cfg, err := getAwsConfig(dsInfo) return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
if err != nil { }
return nil, err
}
sess, err := newSession(cfg) // CloudWatch client factory.
if err != nil { //
return nil, err // Stubbable by tests.
} var newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
client := cloudwatch.New(sess)
client.Handlers.Send.PushFront(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
})
client := cloudwatchlogs.New(sess, cfg) return client
}
// CloudWatch logs client factory.
//
// Stubbable by tests.
var newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
client := cloudwatchlogs.New(sess)
client.Handlers.Send.PushFront(func(r *request.Request) { client.Handlers.Send.PushFront(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
}) })
return client, nil return client
} }
func isTerminated(queryStatus string) bool { // EC2 client factory.
return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout" //
// Stubbable by tests.
var newEC2Client = func(provider client.ConfigProvider) ec2iface.EC2API {
return ec2.New(provider)
}
// RGTA client factory.
//
// Stubbable by tests.
var newRGTAClient = func(provider client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
return resourcegroupstaggingapi.New(provider)
} }

14
pkg/tsdb/cloudwatch/credentials.go
Unescape View File

@ -174,17 +174,3 @@ func ecsCredProvider(sess *session.Session, uri string) credentials.Provider {
func ec2RoleProvider(sess client.ConfigProvider) credentials.Provider { func ec2RoleProvider(sess client.ConfigProvider) credentials.Provider {
return &ec2rolecreds.EC2RoleProvider{Client: newEC2Metadata(sess), ExpiryWindow: 5 * time.Minute} return &ec2rolecreds.EC2RoleProvider{Client: newEC2Metadata(sess), ExpiryWindow: 5 * time.Minute}
} }
func getAwsConfig(dsInfo *datasourceInfo) (*aws.Config, error) {
creds, err := getCredentials(dsInfo)
if err != nil {
return nil, err
}
cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
return cfg, nil
}

510
pkg/tsdb/cloudwatch/log_actions_test.go
Unescape View File

@ -7,7 +7,9 @@ import (
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs" "github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb" "github.com/grafana/grafana/pkg/tsdb"
@ -16,84 +18,229 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
//*** func TestQuery_DescribeLogGroups(t *testing.T) {
// LogActions Tests origNewCWLogsClient := newCWLogsClient
//*** t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsEmpty(t *testing.T) { newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
executor := &cloudWatchExecutor{} return cli
}
logsClient := &FakeLogsClient{ t.Run("Empty log group name prefix", func(t *testing.T) {
Config: aws.Config{ cli = fakeCWLogsClient{
Region: aws.String("default"), logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
}, },
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "DescribeLogGroups",
"limit": 50, "limit": 50,
}),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
},
}),
},
},
}, resp)
}) })
frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params) t.Run("Non-empty log group name prefix", func(t *testing.T) {
cli = fakeCWLogsClient{
expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")}) logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
expectedFrame := data.NewFrame("logGroups", expectedField) LogGroups: []*cloudwatchlogs.LogGroup{
{
assert.Equal(t, nil, err) LogGroupName: aws.String("group_a"),
assert.Equal(t, expectedFrame, frame) },
} {
LogGroupName: aws.String("group_b"),
func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) { },
executor := &cloudWatchExecutor{} {
LogGroupName: aws.String("group_c"),
logsClient := &FakeLogsClient{ },
Config: aws.Config{ },
Region: aws.String("default"),
}, },
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "DescribeLogGroups",
"logGroupNamePrefix": "g", "logGroupNamePrefix": "g",
}),
},
},
}) })
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
},
}),
},
},
}, resp)
})
}
frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params) func TestQuery_GetLogGroupFields(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")}) var cli fakeCWLogsClient
expectedFrame := data.NewFrame("logGroups", expectedField)
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
}
func TestHandleGetLogGroupFields_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) { newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
executor := &cloudWatchExecutor{} return cli
}
logsClient := &FakeLogsClient{ cli = fakeCWLogsClient{
Config: aws.Config{ logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
Region: aws.String("default"), LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
}, },
} }
params := simplejson.NewFromAny(map[string]interface{}{ const refID = "A"
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "GetLogGroupFields",
"logGroupName": "group_a", "logGroupName": "group_a",
"limit": 50, "limit": 50,
}),
},
},
}) })
require.NoError(t, err)
require.NotNil(t, resp)
expFrame := &data.Frame{
Name: refID,
Fields: []*data.Field{
data.NewField("name", nil, []*string{
aws.String("field_a"), aws.String("field_b"), aws.String("field_c"),
}),
data.NewField("percent", nil, []*int64{
aws.Int64(100), aws.Int64(30), aws.Int64(55),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
}
expFrame.RefID = refID
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
refID: {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
RefId: refID,
},
},
}, resp)
}
frame, err := executor.handleGetLogGroupFields(context.Background(), logsClient, params, "A") func TestQuery_StartQuery(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
expectedNameField := data.NewField("name", nil, []*string{aws.String("field_a"), aws.String("field_b"), aws.String("field_c")}) t.Cleanup(func() {
expectedPercentField := data.NewField("percent", nil, []*int64{aws.Int64(100), aws.Int64(30), aws.Int64(55)}) newCWLogsClient = origNewCWLogsClient
expectedFrame := data.NewFrame("A", expectedNameField, expectedPercentField) })
expectedFrame.RefID = "A"
assert.Equal(t, nil, err) var cli fakeCWLogsClient
assert.Equal(t, expectedFrame, frame)
}
func TestExecuteStartQuery(t *testing.T) { newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
executor := &cloudWatchExecutor{} return cli
}
logsClient := &FakeLogsClient{ t.Run("invalid time range", func(t *testing.T) {
Config: aws.Config{ cli = fakeCWLogsClient{
Region: aws.String("default"), logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
}, },
} }
@ -102,26 +249,44 @@ func TestExecuteStartQuery(t *testing.T) {
To: "1584700643000", To: "1584700643000",
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
"region": "default", _, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StartQuery",
"limit": 50, "limit": 50,
"region": "default",
"queryString": "fields @message", "queryString": "fields @message",
}),
},
},
}) })
require.Error(t, err)
response, err := executor.executeStartQuery(context.Background(), logsClient, params, timeRange)
var expectedResponse *cloudwatchlogs.StartQueryOutput = nil
assert.Equal(t, expectedResponse, response)
assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err) assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err)
} })
func TestHandleStartQuery(t *testing.T) {
executor := &cloudWatchExecutor{}
logsClient := &FakeLogsClient{ t.Run("valid time range", func(t *testing.T) {
Config: aws.Config{ const refID = "A"
Region: aws.String("default"), cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
}, },
} }
@ -130,81 +295,198 @@ func TestHandleStartQuery(t *testing.T) {
To: "1584873443000", To: "1584873443000",
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
"region": "default", resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StartQuery",
"limit": 50, "limit": 50,
"region": "default",
"queryString": "fields @message", "queryString": "fields @message",
}),
},
},
}) })
require.NoError(t, err)
frame, err := executor.handleStartQuery(context.Background(), logsClient, params, timeRange, "A") expFrame := data.NewFrame(
refID,
expectedField := data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}) data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}),
expectedFrame := data.NewFrame("A", expectedField) )
expectedFrame.RefID = "A" expFrame.RefID = refID
expectedFrame.Meta = &data.FrameMeta{ expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{ Custom: map[string]interface{}{
"Region": "default", "Region": "default",
}, },
PreferredVisualization: "logs",
} }
assert.Equal(t, &tsdb.Response{
assert.Equal(t, nil, err) Results: map[string]*tsdb.QueryResult{
assert.Equal(t, expectedFrame, frame) refID: {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
RefId: refID,
},
},
}, resp)
})
} }
func TestHandleStopQuery(t *testing.T) { func TestQuery_StopQuery(t *testing.T) {
executor := &cloudWatchExecutor{} origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
logsClient := &FakeLogsClient{ cli = fakeCWLogsClient{
Config: aws.Config{ logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
Region: aws.String("default"), LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
}, },
},
}
timeRange := &tsdb.TimeRange{
From: "1584873443000",
To: "1584700643000",
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StopQuery",
"queryId": "abcd-efgh-ijkl-mnop", "queryId": "abcd-efgh-ijkl-mnop",
}),
},
},
}) })
require.NoError(t, err)
frame, err := executor.handleStopQuery(context.Background(), logsClient, params) expFrame := &data.Frame{
Name: "StopQueryResponse",
Fields: []*data.Field{
data.NewField("success", nil, []bool{true}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
}
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
},
},
}, resp)
}
expectedField := data.NewField("success", nil, []bool{true}) func TestQuery_GetQueryResults(t *testing.T) {
expectedFrame := data.NewFrame("StopQueryResponse", expectedField) origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
assert.Equal(t, nil, err) var cli fakeCWLogsClient
assert.Equal(t, expectedFrame, frame)
}
func TestHandleGetQueryResults(t *testing.T) { newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
executor := &cloudWatchExecutor{} return cli
}
logsClient := &FakeLogsClient{ const refID = "A"
Config: aws.Config{ cli = fakeCWLogsClient{
Region: aws.String("default"), queryResults: cloudwatchlogs.GetQueryResultsOutput{
Results: [][]*cloudwatchlogs.ResultField{
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:37:23.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_1"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("abcdefg"),
},
},
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:40:43.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_2"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("hijklmnop"),
},
},
},
Statistics: &cloudwatchlogs.QueryStatistics{
BytesScanned: aws.Float64(512),
RecordsMatched: aws.Float64(256),
RecordsScanned: aws.Float64(1024),
},
Status: aws.String("Complete"),
}, },
} }
params := simplejson.NewFromAny(map[string]interface{}{ executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "GetQueryResults",
"queryId": "abcd-efgh-ijkl-mnop", "queryId": "abcd-efgh-ijkl-mnop",
}),
},
},
}) })
frame, err := executor.handleGetQueryResults(context.Background(), logsClient, params, "A")
require.NoError(t, err) require.NoError(t, err)
timeA, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
time1, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
require.NoError(t, err) require.NoError(t, err)
timeB, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000") time2, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000")
require.NoError(t, err) require.NoError(t, err)
expectedTimeField := data.NewField("@timestamp", nil, []*time.Time{ expField1 := data.NewField("@timestamp", nil, []*time.Time{
aws.Time(timeA), aws.Time(timeB), aws.Time(time1), aws.Time(time2),
}) })
expectedTimeField.SetConfig(&data.FieldConfig{DisplayName: "Time"}) expField1.SetConfig(&data.FieldConfig{DisplayName: "Time"})
expField2 := data.NewField("field_b", nil, []*string{
expectedFieldB := data.NewField("field_b", nil, []*string{
aws.String("b_1"), aws.String("b_2"), aws.String("b_1"), aws.String("b_2"),
}) })
expFrame := data.NewFrame(refID, expField1, expField2)
expectedFrame := data.NewFrame("A", expectedTimeField, expectedFieldB) expFrame.RefID = refID
expectedFrame.RefID = "A" expFrame.Meta = &data.FrameMeta{
expectedFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{ Custom: map[string]interface{}{
"Status": "Complete", "Status": "Complete",
"Statistics": cloudwatchlogs.QueryStatistics{ "Statistics": cloudwatchlogs.QueryStatistics{
@ -213,9 +495,15 @@ func TestHandleGetQueryResults(t *testing.T) {
RecordsScanned: aws.Float64(1024), RecordsScanned: aws.Float64(1024),
}, },
}, },
PreferredVisualization: "logs",
} }
assert.Equal(t, nil, err) assert.Equal(t, &tsdb.Response{
assert.ElementsMatch(t, expectedFrame.Fields, frame.Fields) Results: map[string]*tsdb.QueryResult{
assert.Equal(t, expectedFrame.Meta, frame.Meta) refID: {
RefId: refID,
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
},
},
}, resp)
} }

106
pkg/tsdb/cloudwatch/logs_client_mock.go
Unescape View File

@ -1,106 +0,0 @@
package cloudwatch
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
)
type FakeLogsClient struct {
cloudwatchlogsiface.CloudWatchLogsAPI
Config aws.Config
}
func (f FakeLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
return &cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
}, nil
}
func (f FakeLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
return &cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
}, nil
}
func (f FakeLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
return &cloudwatchlogs.StartQueryOutput{
QueryId: aws.String("abcd-efgh-ijkl-mnop"),
}, nil
}
func (f FakeLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
return &cloudwatchlogs.StopQueryOutput{
Success: aws.Bool(true),
}, nil
}
func (f FakeLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
return &cloudwatchlogs.GetQueryResultsOutput{
Results: [][]*cloudwatchlogs.ResultField{
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:37:23.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_1"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("abcdefg"),
},
},
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:40:43.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_2"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("hijklmnop"),
},
},
},
Statistics: &cloudwatchlogs.QueryStatistics{
BytesScanned: aws.Float64(512),
RecordsMatched: aws.Float64(256),
RecordsScanned: aws.Float64(1024),
},
Status: aws.String("Complete"),
}, nil
}

97
pkg/tsdb/cloudwatch/metric_find_query.go
Unescape View File

@ -12,7 +12,6 @@ import (
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
@ -310,8 +309,7 @@ func parseMultiSelectValue(input string) []string {
// Whenever this list is updated, the frontend list should also be updated. // Whenever this list is updated, the frontend list should also be updated.
// Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
const region = "default" dsInfo := e.getDSInfo(defaultRegion)
dsInfo := e.getDSInfo(region)
profile := dsInfo.Profile profile := dsInfo.Profile
if cache, ok := regionCache.Load(profile); ok { if cache, ok := regionCache.Load(profile); ok {
if cache2, ok2 := cache.([]suggestData); ok2 { if cache2, ok2 := cache.([]suggestData); ok2 {
@ -319,12 +317,12 @@ func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s
} }
} }
err := e.ensureClientSession("default") client, err := e.getEC2Client(defaultRegion)
if err != nil { if err != nil {
return nil, err return nil, err
} }
regions := knownRegions regions := knownRegions
r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{}) r, err := client.DescribeRegions(&ec2.DescribeRegionsInput{})
if err != nil { if err != nil {
// ignore error for backward compatibility // ignore error for backward compatibility
plog.Error("Failed to get regions", "error", err) plog.Error("Failed to get regions", "error", err)
@ -389,7 +387,7 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s
dsInfo := e.getDSInfo(region) dsInfo := e.getDSInfo(region)
dsInfo.Namespace = namespace dsInfo.Namespace = namespace
if namespaceMetrics, err = e.getMetricsForCustomMetrics(region, e.getAllMetrics); err != nil { if namespaceMetrics, err = e.getMetricsForCustomMetrics(region); err != nil {
return nil, errors.New("Unable to call AWS API") return nil, errors.New("Unable to call AWS API")
} }
} }
@ -418,7 +416,7 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters
dsInfo := e.getDSInfo(region) dsInfo := e.getDSInfo(region)
dsInfo.Namespace = namespace dsInfo.Namespace = namespace
if dimensionValues, err = e.getDimensionsForCustomMetrics(region, e.getAllMetrics); err != nil { if dimensionValues, err = e.getDimensionsForCustomMetrics(region); err != nil {
return nil, errors.New("Unable to call AWS API") return nil, errors.New("Unable to call AWS API")
} }
} }
@ -483,31 +481,10 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
return result, nil return result, nil
} }
func (e *cloudWatchExecutor) ensureClientSession(region string) error {
if e.ec2Svc == nil {
dsInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(dsInfo)
if err != nil {
return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
}
sess, err := session.NewSession(cfg)
if err != nil {
return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
}
e.ec2Svc = ec2.New(sess, cfg)
}
return nil
}
func (e *cloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { func (e *cloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
region := parameters.Get("region").MustString() region := parameters.Get("region").MustString()
instanceId := parameters.Get("instanceId").MustString() instanceId := parameters.Get("instanceId").MustString()
err := e.ensureClientSession(region)
if err != nil {
return nil, err
}
instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId)) instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId))
instances, err := e.ec2DescribeInstances(region, nil, instanceIds) instances, err := e.ec2DescribeInstances(region, nil, instanceIds)
if err != nil { if err != nil {
@ -547,11 +524,6 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
} }
} }
err := e.ensureClientSession(region)
if err != nil {
return nil, err
}
instances, err := e.ec2DescribeInstances(region, filters, nil) instances, err := e.ec2DescribeInstances(region, filters, nil)
if err != nil { if err != nil {
return nil, err return nil, err
@ -610,32 +582,11 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
return result, nil return result, nil
} }
func (e *cloudWatchExecutor) ensureRGTAClientSession(region string) error {
if e.rgtaSvc == nil {
dsInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(dsInfo)
if err != nil {
return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
}
sess, err := session.NewSession(cfg)
if err != nil {
return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
}
e.rgtaSvc = resourcegroupstaggingapi.New(sess, cfg)
}
return nil
}
func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) { func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
region := parameters.Get("region").MustString() region := parameters.Get("region").MustString()
resourceType := parameters.Get("resourceType").MustString() resourceType := parameters.Get("resourceType").MustString()
filterJson := parameters.Get("tags").MustMap() filterJson := parameters.Get("tags").MustMap()
err := e.ensureRGTAClientSession(region)
if err != nil {
return nil, err
}
var filters []*resourcegroupstaggingapi.TagFilter var filters []*resourcegroupstaggingapi.TagFilter
for k, v := range filterJson { for k, v := range filterJson {
if vv, ok := v.([]interface{}); ok { if vv, ok := v.([]interface{}); ok {
@ -706,8 +657,13 @@ func (e *cloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.
InstanceIds: instanceIds, InstanceIds: instanceIds,
} }
client, err := e.getEC2Client(region)
if err != nil {
return nil, err
}
var resp ec2.DescribeInstancesOutput var resp ec2.DescribeInstancesOutput
if err := e.ec2Svc.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool { if err := client.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
resp.Reservations = append(resp.Reservations, page.Reservations...) resp.Reservations = append(resp.Reservations, page.Reservations...)
return !lastPage return !lastPage
}); err != nil { }); err != nil {
@ -724,8 +680,13 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
TagFilters: filters, TagFilters: filters,
} }
client, err := e.getRGTAClient(region)
if err != nil {
return nil, err
}
var resp resourcegroupstaggingapi.GetResourcesOutput var resp resourcegroupstaggingapi.GetResourcesOutput
if err := e.rgtaSvc.GetResourcesPages(params, if err := client.GetResourcesPages(params,
func(page *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { func(page *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool {
resp.ResourceTagMappingList = append(resp.ResourceTagMappingList, page.ResourceTagMappingList...) resp.ResourceTagMappingList = append(resp.ResourceTagMappingList, page.ResourceTagMappingList...)
return !lastPage return !lastPage
@ -737,27 +698,18 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
} }
func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetricsOutput, error) { func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetricsOutput, error) {
dsInfo := e.getDSInfo(region) client, err := e.getCWClient(region)
creds, err := getCredentials(dsInfo)
if err != nil { if err != nil {
return cloudwatch.ListMetricsOutput{}, err return cloudwatch.ListMetricsOutput{}, err
} }
cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
sess, err := session.NewSession(cfg)
if err != nil {
return cloudwatch.ListMetricsOutput{}, err
}
svc := cloudwatch.New(sess, cfg)
dsInfo := e.getDSInfo(region)
params := &cloudwatch.ListMetricsInput{ params := &cloudwatch.ListMetricsInput{
Namespace: aws.String(dsInfo.Namespace), Namespace: aws.String(dsInfo.Namespace),
} }
var resp cloudwatch.ListMetricsOutput var resp cloudwatch.ListMetricsOutput
err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
metrics.MAwsCloudWatchListMetrics.Inc() metrics.MAwsCloudWatchListMetrics.Inc()
metrics, err := awsutil.ValuesAtPath(page, "Metrics") metrics, err := awsutil.ValuesAtPath(page, "Metrics")
if err != nil { if err != nil {
@ -769,12 +721,13 @@ func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetric
} }
return !lastPage return !lastPage
}) })
return resp, err return resp, err
} }
var metricsCacheLock sync.Mutex var metricsCacheLock sync.Mutex
func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string) ([]string, error) {
metricsCacheLock.Lock() metricsCacheLock.Lock()
defer metricsCacheLock.Unlock() defer metricsCacheLock.Unlock()
@ -794,7 +747,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
} }
result, err := getAllMetrics(region) result, err := e.getAllMetrics(region)
if err != nil { if err != nil {
return []string{}, err return []string{}, err
} }
@ -813,7 +766,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
var dimensionsCacheLock sync.Mutex var dimensionsCacheLock sync.Mutex
func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) { func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string) ([]string, error) {
dimensionsCacheLock.Lock() dimensionsCacheLock.Lock()
defer dimensionsCacheLock.Unlock() defer dimensionsCacheLock.Unlock()
@ -833,7 +786,7 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAll
if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) { if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
} }
result, err := getAllMetrics(region) result, err := e.getAllMetrics(region)
if err != nil { if err != nil {
return []string{}, err return []string{}, err
} }

501
pkg/tsdb/cloudwatch/metric_find_query_test.go
Unescape View File

@ -5,57 +5,35 @@ import (
"testing" "testing"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb" "github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
type mockedEc2 struct { func TestQuery_Metrics(t *testing.T) {
ec2iface.EC2API origNewCWClient := newCWClient
Resp ec2.DescribeInstancesOutput t.Cleanup(func() {
RespRegions ec2.DescribeRegionsOutput newCWClient = origNewCWClient
} })
type mockedRGTA struct {
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
Resp resourcegroupstaggingapi.GetResourcesOutput
}
func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
fn(&m.Resp, true)
return nil
}
func (m mockedEc2) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
return &m.RespRegions, nil
}
func (m mockedRGTA) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput, fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error { var client fakeCWClient
fn(&m.Resp, true)
return nil
}
func TestCloudWatchMetrics(t *testing.T) { newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
t.Run("When calling getMetricsForCustomMetrics", func(t *testing.T) { return client
const region = "us-east-1"
e := &cloudWatchExecutor{
DataSource: &models.DataSource{
Database: "default",
JsonData: simplejson.NewFromAny(map[string]interface{}{
"Region": region,
}),
},
} }
f := func(region string) (cloudwatch.ListMetricsOutput, error) {
return cloudwatch.ListMetricsOutput{ t.Run("Custom metrics", func(t *testing.T) {
Metrics: []*cloudwatch.Metric{ client = fakeCWClient{
metrics: []*cloudwatch.Metric{
{ {
MetricName: aws.String("Test_MetricName"), MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{ Dimensions: []*cloudwatch.Dimension{
@ -65,27 +43,54 @@ func TestCloudWatchMetrics(t *testing.T) {
}, },
}, },
}, },
}, nil
} }
metrics, err := e.getMetricsForCustomMetrics(region, f) executor := newExecutor()
require.NoError(t, err) resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
assert.Contains(t, metrics, "Test_MetricName") {
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "metrics",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
}) })
require.NoError(t, err)
t.Run("When calling getDimensionsForCustomMetrics", func(t *testing.T) { assert.Equal(t, &tsdb.Response{
const region = "us-east-1" Results: map[string]*tsdb.QueryResult{
e := &cloudWatchExecutor{ "": {
DataSource: &models.DataSource{ Meta: simplejson.NewFromAny(map[string]interface{}{
Database: "default", "rowCount": 1,
JsonData: simplejson.NewFromAny(map[string]interface{}{
"Region": region,
}), }),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
}, },
} {
f := func(region string) (cloudwatch.ListMetricsOutput, error) { Text: "value",
return cloudwatch.ListMetricsOutput{ },
Metrics: []*cloudwatch.Metric{ },
Rows: []tsdb.RowValues{
{
"Test_MetricName",
"Test_MetricName",
},
},
},
},
},
},
}, resp)
})
t.Run("Dimension keys for custom metrics", func(t *testing.T) {
client = fakeCWClient{
metrics: []*cloudwatch.Metric{
{ {
MetricName: aws.String("Test_MetricName"), MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{ Dimensions: []*cloudwatch.Dimension{
@ -95,48 +100,140 @@ func TestCloudWatchMetrics(t *testing.T) {
}, },
}, },
}, },
}, nil
} }
dimensionKeys, err := e.getDimensionsForCustomMetrics(region, f) executor := newExecutor()
require.NoError(t, err) resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
assert.Contains(t, dimensionKeys, "Test_DimensionName") {
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
}) })
require.NoError(t, err)
t.Run("When calling handleGetRegions", func(t *testing.T) { assert.Equal(t, &tsdb.Response{
executor := &cloudWatchExecutor{ Results: map[string]*tsdb.QueryResult{
ec2Svc: mockedEc2{RespRegions: ec2.DescribeRegionsOutput{ "": {
Regions: []*ec2.Region{ Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{ {
RegionName: aws.String("ap-northeast-2"), Text: "text",
}, },
{
Text: "value",
}, },
}}, },
} Rows: []tsdb.RowValues{
jsonData := simplejson.New() {
jsonData.Set("defaultRegion", "default") "Test_DimensionName",
executor.DataSource = &models.DataSource{ "Test_DimensionName",
JsonData: jsonData, },
SecureJsonData: securejsondata.SecureJsonData{}, },
},
},
},
},
}, resp)
})
}
func TestQuery_Regions(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
})
var cli fakeEC2Client
newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
return cli
} }
result, err := executor.handleGetRegions(context.Background(), simplejson.New(), &tsdb.TsdbQuery{}) t.Run("An extra region", func(t *testing.T) {
const regionName = "xtra-region"
cli = fakeEC2Client{
regions: []string{regionName},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "regions",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "af-south-1", result[0].Text) rows := []tsdb.RowValues{}
assert.Equal(t, "ap-east-1", result[1].Text) for _, region := range knownRegions {
assert.Equal(t, "ap-northeast-1", result[2].Text) rows = append(rows, []interface{}{
assert.Equal(t, "ap-northeast-2", result[3].Text) region,
region,
})
}
rows = append(rows, []interface{}{
regionName,
regionName,
})
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": len(knownRegions) + 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: rows,
},
},
},
},
}, resp)
})
}
func TestQuery_InstanceAttributes(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
}) })
t.Run("When calling handleGetEc2InstanceAttribute", func(t *testing.T) { var cli fakeEC2Client
executor := &cloudWatchExecutor{
ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
Reservations: []*ec2.Reservation{ return cli
}
t.Run("Get instance ID", func(t *testing.T) {
const instanceID = "i-12345678"
cli = fakeEC2Client{
reservations: []*ec2.Reservation{
{ {
Instances: []*ec2.Instance{ Instances: []*ec2.Instance{
{ {
InstanceId: aws.String("i-12345678"), InstanceId: aws.String(instanceID),
Tags: []*ec2.Tag{ Tags: []*ec2.Tag{
{ {
Key: aws.String("Environment"), Key: aws.String("Environment"),
@ -147,25 +244,72 @@ func TestCloudWatchMetrics(t *testing.T) {
}, },
}, },
}, },
}},
} }
executor := newExecutor()
json := simplejson.New() resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
json.Set("region", "us-east-1") Queries: []*tsdb.Query{
json.Set("attributeName", "InstanceId") {
filters := make(map[string]interface{}) Model: simplejson.NewFromAny(map[string]interface{}{
filters["tag:Environment"] = []string{"production"} "type": "metricFindQuery",
json.Set("filters", filters) "subtype": "ec2_instance_attribute",
result, err := executor.handleGetEc2InstanceAttribute(context.Background(), json, &tsdb.TsdbQuery{}) "region": "us-east-1",
"attributeName": "InstanceId",
"filters": map[string]interface{}{
"tag:Environment": []string{"production"},
},
}),
},
},
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "i-12345678", result[0].Text) assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
instanceID,
instanceID,
},
},
},
},
},
},
}, resp)
})
}
func TestQuery_EBSVolumeIDs(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
}) })
t.Run("When calling handleGetEbsVolumeIds", func(t *testing.T) { var cli fakeEC2Client
executor := &cloudWatchExecutor{
ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{ newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
Reservations: []*ec2.Reservation{ return cli
}
t.Run("", func(t *testing.T) {
const instanceIDs = "{i-1, i-2, i-3}"
cli = fakeEC2Client{
reservations: []*ec2.Reservation{
{ {
Instances: []*ec2.Instance{ Instances: []*ec2.Instance{
{ {
@ -203,31 +347,87 @@ func TestCloudWatchMetrics(t *testing.T) {
}, },
}, },
}, },
}},
} }
executor := newExecutor()
json := simplejson.New() resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
json.Set("region", "us-east-1") Queries: []*tsdb.Query{
json.Set("instanceId", "{i-1, i-2, i-3, i-4}") {
result, err := executor.handleGetEbsVolumeIds(context.Background(), json, &tsdb.TsdbQuery{}) Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "ebs_volume_ids",
"region": "us-east-1",
"instanceId": instanceIDs,
}),
},
},
})
require.NoError(t, err) require.NoError(t, err)
require.Len(t, result, 8) assert.Equal(t, &tsdb.Response{
assert.Equal(t, "vol-1-1", result[0].Text) Results: map[string]*tsdb.QueryResult{
assert.Equal(t, "vol-1-2", result[1].Text) "": {
assert.Equal(t, "vol-2-1", result[2].Text) Meta: simplejson.NewFromAny(map[string]interface{}{
assert.Equal(t, "vol-2-2", result[3].Text) "rowCount": 6,
assert.Equal(t, "vol-3-1", result[4].Text) }),
assert.Equal(t, "vol-3-2", result[5].Text) Tables: []*tsdb.Table{
assert.Equal(t, "vol-4-1", result[6].Text) {
assert.Equal(t, "vol-4-2", result[7].Text) Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"vol-1-1",
"vol-1-1",
},
{
"vol-1-2",
"vol-1-2",
},
{
"vol-2-1",
"vol-2-1",
},
{
"vol-2-2",
"vol-2-2",
},
{
"vol-3-1",
"vol-3-1",
},
{
"vol-3-2",
"vol-3-2",
},
},
},
},
},
},
}, resp)
})
}
func TestQuery_ResourceARNs(t *testing.T) {
origNewRGTAClient := newRGTAClient
t.Cleanup(func() {
newRGTAClient = origNewRGTAClient
}) })
t.Run("When calling handleGetResourceArns", func(t *testing.T) { var cli fakeRGTAClient
executor := &cloudWatchExecutor{
rgtaSvc: mockedRGTA{ newRGTAClient = func(client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
Resp: resourcegroupstaggingapi.GetResourcesOutput{ return cli
ResourceTagMappingList: []*resourcegroupstaggingapi.ResourceTagMapping{ }
t.Run("", func(t *testing.T) {
cli = fakeRGTAClient{
tagMapping: []*resourcegroupstaggingapi.ResourceTagMapping{
{ {
ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"), ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"),
Tags: []*resourcegroupstaggingapi.Tag{ Tags: []*resourcegroupstaggingapi.Tag{
@ -247,36 +447,55 @@ func TestCloudWatchMetrics(t *testing.T) {
}, },
}, },
}, },
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "resource_arns",
"region": "us-east-1",
"resourceType": "ec2:instance",
"tags": map[string]interface{}{
"Environment": []string{"production"},
}, },
}),
}, },
} },
})
json := simplejson.New()
json.Set("region", "us-east-1")
json.Set("resourceType", "ec2:instance")
tags := make(map[string]interface{})
tags["Environment"] = []string{"production"}
json.Set("tags", tags)
result, err := executor.handleGetResourceArns(context.Background(), json, &tsdb.TsdbQuery{})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Text) assert.Equal(t, &tsdb.Response{
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Value) Results: map[string]*tsdb.QueryResult{
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Text) "": {
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Value) Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 2,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
},
{
"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
},
},
},
},
},
},
}, resp)
}) })
} }
func TestParseMultiSelectValue(t *testing.T) {
values := parseMultiSelectValue(" i-someInstance ")
assert.Equal(t, []string{"i-someInstance"}, values)
values = parseMultiSelectValue("{i-05}")
assert.Equal(t, []string{"i-05"}, values)
values = parseMultiSelectValue(" {i-01, i-03, i-04} ")
assert.Equal(t, []string{"i-01", "i-03", "i-04"}, values)
values = parseMultiSelectValue("i-{01}")
assert.Equal(t, []string{"i-{01}"}, values)
}

2
pkg/tsdb/cloudwatch/query_transformer_test.go
Unescape View File

@ -10,7 +10,7 @@ import (
func TestQueryTransformer(t *testing.T) { func TestQueryTransformer(t *testing.T) {
Convey("TestQueryTransformer", t, func() { Convey("TestQueryTransformer", t, func() {
Convey("when transforming queries", func() { Convey("when transforming queries", func() {
executor := &cloudWatchExecutor{} executor := newExecutor()
Convey("one cloudwatchQuery is generated when its request query has one stat", func() { Convey("one cloudwatchQuery is generated when its request query has one stat", func() {
requestQueries := []*requestQuery{ requestQueries := []*requestQuery{
{ {

133
pkg/tsdb/cloudwatch/test_utils.go
Unescape View File

@ -0,0 +1,133 @@
package cloudwatch
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
)
func fakeDataSource() *models.DataSource {
jsonData := simplejson.New()
jsonData.Set("defaultRegion", "default")
return &models.DataSource{
Id: 1,
Database: "default",
JsonData: jsonData,
SecureJsonData: securejsondata.SecureJsonData{},
}
}
type fakeCWLogsClient struct {
cloudwatchlogsiface.CloudWatchLogsAPI
logGroups cloudwatchlogs.DescribeLogGroupsOutput
logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput
queryResults cloudwatchlogs.GetQueryResultsOutput
}
func (m fakeCWLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
return &m.queryResults, nil
}
func (m fakeCWLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
return &cloudwatchlogs.StartQueryOutput{
QueryId: aws.String("abcd-efgh-ijkl-mnop"),
}, nil
}
func (m fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
return &cloudwatchlogs.StopQueryOutput{
Success: aws.Bool(true),
}, nil
}
func (m fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
return &m.logGroups, nil
}
func (m fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
return &m.logGroupFields, nil
}
type fakeCWClient struct {
cloudwatchiface.CloudWatchAPI
metrics []*cloudwatch.Metric
}
func (c fakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
fn(&cloudwatch.ListMetricsOutput{
Metrics: c.metrics,
}, true)
return nil
}
type fakeEC2Client struct {
ec2iface.EC2API
regions []string
reservations []*ec2.Reservation
}
func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
regions := []*ec2.Region{}
for _, region := range c.regions {
regions = append(regions, &ec2.Region{
RegionName: aws.String(region),
})
}
return &ec2.DescribeRegionsOutput{
Regions: regions,
}, nil
}
func (c fakeEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput,
fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
reservations := []*ec2.Reservation{}
for _, r := range c.reservations {
instances := []*ec2.Instance{}
for _, inst := range r.Instances {
if len(in.InstanceIds) == 0 {
instances = append(instances, inst)
continue
}
for _, id := range in.InstanceIds {
if *inst.InstanceId == *id {
instances = append(instances, inst)
}
}
}
reservation := &ec2.Reservation{Instances: instances}
reservations = append(reservations, reservation)
}
fn(&ec2.DescribeInstancesOutput{
Reservations: reservations,
}, true)
return nil
}
type fakeRGTAClient struct {
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
tagMapping []*resourcegroupstaggingapi.ResourceTagMapping
}
func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput,
fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error {
fn(&resourcegroupstaggingapi.GetResourcesOutput{
ResourceTagMappingList: c.tagMapping,
}, true)
return nil
}

2
pkg/tsdb/cloudwatch/time_series_query_test.go
Unescape View File

@ -9,7 +9,7 @@ import (
) )
func TestTimeSeriesQuery(t *testing.T) { func TestTimeSeriesQuery(t *testing.T) {
executor := &cloudWatchExecutor{} executor := newExecutor()
t.Run("End time before start time should result in error", func(t *testing.T) { t.Run("End time before start time should result in error", func(t *testing.T) {
_, err := executor.executeTimeSeriesQuery(context.TODO(), &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")}) _, err := executor.executeTimeSeriesQuery(context.TODO(), &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")})

Write Preview
Loading…
Cancel
Save