cloudwatch: Consolidate client logic (#25555)

* cloudwatch: Consolidate client logic

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

pkg/tsdb/cloudwatch/cloudwatch.go

@@ -9,11 +9,17 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -45,75 +51,95 @@ var plog = log.New("tsdb.cloudwatch")
 var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
 
 func init() {
-	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", newcloudWatchExecutor)
+	tsdb.RegisterTsdbQueryEndpoint("cloudwatch", func(ds *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
+		return newExecutor(), nil
+	})
 }
 
-func newcloudWatchExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
-	e := &cloudWatchExecutor{
-		DataSource: datasource,
+func newExecutor() *cloudWatchExecutor {
+	return &cloudWatchExecutor{
+		logsClientsByRegion: map[string]cloudwatchlogsiface.CloudWatchLogsAPI{},
 	}
-
-	dsInfo := e.getDSInfo(defaultRegion)
-	defaultLogsClient, err := retrieveLogsClient(dsInfo)
-	if err != nil {
-		return nil, err
-	}
-	e.logsClientsByRegion = map[string]*cloudwatchlogs.CloudWatchLogs{
-		dsInfo.Region: defaultLogsClient,
-		defaultRegion: defaultLogsClient,
-	}
-
-	return e, nil
 }
 
 // cloudWatchExecutor executes CloudWatch requests.
 type cloudWatchExecutor struct {
 	*models.DataSource
-	ec2Svc  ec2iface.EC2API
-	rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
 
-	logsClientsByRegion map[string](*cloudwatchlogs.CloudWatchLogs)
-	mux                 sync.Mutex
+	ec2Client           ec2iface.EC2API
+	rgtaClient          resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
+	logsClientsByRegion map[string]cloudwatchlogsiface.CloudWatchLogsAPI
+	mtx                 sync.Mutex
 }
 
-func (e *cloudWatchExecutor) getCWClient(region string) (*cloudwatch.CloudWatch, error) {
-	datasourceInfo := e.getDSInfo(region)
-	cfg, err := getAwsConfig(datasourceInfo)
+func (e *cloudWatchExecutor) newSession(region string) (*session.Session, error) {
+	dsInfo := e.getDSInfo(region)
+	creds, err := getCredentials(dsInfo)
 	if err != nil {
 		return nil, err
 	}
 
-	sess, err := newSession(cfg)
+	cfg := &aws.Config{
+		Region:      aws.String(dsInfo.Region),
+		Credentials: creds,
+	}
+	return newSession(cfg)
+}
+
+func (e *cloudWatchExecutor) getCWClient(region string) (cloudwatchiface.CloudWatchAPI, error) {
+	sess, err := e.newSession(region)
 	if err != nil {
 		return nil, err
 	}
-
-	client := cloudwatch.New(sess, cfg)
-
-	client.Handlers.Send.PushFront(func(r *request.Request) {
-		r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
-	})
-
-	return client, nil
+	return newCWClient(sess), nil
 }
 
-func (e *cloudWatchExecutor) getCWLogsClient(region string) (*cloudwatchlogs.CloudWatchLogs, error) {
-	e.mux.Lock()
-	defer e.mux.Unlock()
+func (e *cloudWatchExecutor) getCWLogsClient(region string) (cloudwatchlogsiface.CloudWatchLogsAPI, error) {
+	e.mtx.Lock()
+	defer e.mtx.Unlock()
 
 	if logsClient, ok := e.logsClientsByRegion[region]; ok {
 		return logsClient, nil
 	}
 
-	dsInfo := e.getDSInfo(region)
-	newLogsClient, err := retrieveLogsClient(dsInfo)
+	sess, err := e.newSession(region)
 	if err != nil {
 		return nil, err
 	}
 
-	e.logsClientsByRegion[region] = newLogsClient
+	logsClient := newCWLogsClient(sess)
+	e.logsClientsByRegion[region] = logsClient
+
+	return logsClient, nil
+}
+
+func (e *cloudWatchExecutor) getEC2Client(region string) (ec2iface.EC2API, error) {
+	if e.ec2Client != nil {
+		return e.ec2Client, nil
+	}
+
+	sess, err := e.newSession(region)
+	if err != nil {
+		return nil, err
+	}
+	e.ec2Client = newEC2Client(sess)
 
-	return newLogsClient, nil
+	return e.ec2Client, nil
+}
+
+func (e *cloudWatchExecutor) getRGTAClient(region string) (resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI,
+	error) {
+	if e.rgtaClient != nil {
+		return e.rgtaClient, nil
+	}
+
+	sess, err := e.newSession(region)
+	if err != nil {
+		return nil, err
+	}
+	e.rgtaClient = newRGTAClient(sess)
+
+	return e.rgtaClient, nil
 }
 
 func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
@@ -279,26 +305,44 @@ func (e *cloudWatchExecutor) getDSInfo(region string) *datasourceInfo {
 	}
 }
 
-func retrieveLogsClient(dsInfo *datasourceInfo) (*cloudwatchlogs.CloudWatchLogs, error) {
-	cfg, err := getAwsConfig(dsInfo)
-	if err != nil {
-		return nil, err
-	}
+func isTerminated(queryStatus string) bool {
+	return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
+}
 
-	sess, err := newSession(cfg)
-	if err != nil {
-		return nil, err
-	}
+// CloudWatch client factory.
+//
+// Stubbable by tests.
+var newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
+	client := cloudwatch.New(sess)
+	client.Handlers.Send.PushFront(func(r *request.Request) {
+		r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
+	})
 
-	client := cloudwatchlogs.New(sess, cfg)
+	return client
+}
 
+// CloudWatch logs client factory.
+//
+// Stubbable by tests.
+var newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+	client := cloudwatchlogs.New(sess)
 	client.Handlers.Send.PushFront(func(r *request.Request) {
 		r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
 	})
 
-	return client, nil
+	return client
 }
 
-func isTerminated(queryStatus string) bool {
-	return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
+// EC2 client factory.
+//
+// Stubbable by tests.
+var newEC2Client = func(provider client.ConfigProvider) ec2iface.EC2API {
+	return ec2.New(provider)
+}
+
+// RGTA client factory.
+//
+// Stubbable by tests.
+var newRGTAClient = func(provider client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
+	return resourcegroupstaggingapi.New(provider)
 }

pkg/tsdb/cloudwatch/credentials.go

@@ -174,17 +174,3 @@ func ecsCredProvider(sess *session.Session, uri string) credentials.Provider {
 func ec2RoleProvider(sess client.ConfigProvider) credentials.Provider {
 	return &ec2rolecreds.EC2RoleProvider{Client: newEC2Metadata(sess), ExpiryWindow: 5 * time.Minute}
 }
-
-func getAwsConfig(dsInfo *datasourceInfo) (*aws.Config, error) {
-	creds, err := getCredentials(dsInfo)
-	if err != nil {
-		return nil, err
-	}
-
-	cfg := &aws.Config{
-		Region:      aws.String(dsInfo.Region),
-		Credentials: creds,
-	}
-
-	return cfg, nil
-}

pkg/tsdb/cloudwatch/log_actions_test.go

@@ -7,7 +7,9 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/tsdb"
@@ -16,195 +18,475 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-//***
-// LogActions Tests
-//***
+func TestQuery_DescribeLogGroups(t *testing.T) {
+	origNewCWLogsClient := newCWLogsClient
+	t.Cleanup(func() {
+		newCWLogsClient = origNewCWLogsClient
+	})
 
-func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsEmpty(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	var cli fakeCWLogsClient
 
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
-		},
+	newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return cli
 	}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"limit": 50,
+	t.Run("Empty log group name prefix", func(t *testing.T) {
+		cli = fakeCWLogsClient{
+			logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
+				LogGroups: []*cloudwatchlogs.LogGroup{
+					{
+						LogGroupName: aws.String("group_a"),
+					},
+					{
+						LogGroupName: aws.String("group_b"),
+					},
+					{
+						LogGroupName: aws.String("group_c"),
+					},
+				},
+			},
+		}
+
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":    "logAction",
+						"subtype": "DescribeLogGroups",
+						"limit":   50,
+					}),
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
+						&data.Frame{
+							Name: "logGroups",
+							Fields: []*data.Field{
+								data.NewField("logGroupName", nil, []*string{
+									aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
+								}),
+							},
+							Meta: &data.FrameMeta{
+								PreferredVisualization: "logs",
+							},
+						},
+					}),
+				},
+			},
+		}, resp)
 	})
 
-	frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params)
+	t.Run("Non-empty log group name prefix", func(t *testing.T) {
+		cli = fakeCWLogsClient{
+			logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
+				LogGroups: []*cloudwatchlogs.LogGroup{
+					{
+						LogGroupName: aws.String("group_a"),
+					},
+					{
+						LogGroupName: aws.String("group_b"),
+					},
+					{
+						LogGroupName: aws.String("group_c"),
+					},
+				},
+			},
+		}
+
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":               "logAction",
+						"subtype":            "DescribeLogGroups",
+						"logGroupNamePrefix": "g",
+					}),
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
+						&data.Frame{
+							Name: "logGroups",
+							Fields: []*data.Field{
+								data.NewField("logGroupName", nil, []*string{
+									aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
+								}),
+							},
+							Meta: &data.FrameMeta{
+								PreferredVisualization: "logs",
+							},
+						},
+					}),
+				},
+			},
+		}, resp)
+	})
+}
 
-	expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")})
-	expectedFrame := data.NewFrame("logGroups", expectedField)
+func TestQuery_GetLogGroupFields(t *testing.T) {
+	origNewCWLogsClient := newCWLogsClient
+	t.Cleanup(func() {
+		newCWLogsClient = origNewCWLogsClient
+	})
 
-	assert.Equal(t, nil, err)
-	assert.Equal(t, expectedFrame, frame)
-}
+	var cli fakeCWLogsClient
 
-func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return cli
+	}
 
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
+	cli = fakeCWLogsClient{
+		logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
+			LogGroupFields: []*cloudwatchlogs.LogGroupField{
+				{
+					Name:    aws.String("field_a"),
+					Percent: aws.Int64(100),
+				},
+				{
+					Name:    aws.String("field_b"),
+					Percent: aws.Int64(30),
+				},
+				{
+					Name:    aws.String("field_c"),
+					Percent: aws.Int64(55),
+				},
+			},
 		},
 	}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"logGroupNamePrefix": "g",
+	const refID = "A"
+
+	executor := newExecutor()
+	resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+		Queries: []*tsdb.Query{
+			{
+				RefId: refID,
+				Model: simplejson.NewFromAny(map[string]interface{}{
+					"type":         "logAction",
+					"subtype":      "GetLogGroupFields",
+					"logGroupName": "group_a",
+					"limit":        50,
+				}),
+			},
+		},
 	})
-
-	frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params)
-
-	expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")})
-	expectedFrame := data.NewFrame("logGroups", expectedField)
-	assert.Equal(t, nil, err)
-	assert.Equal(t, expectedFrame, frame)
-}
-
-func TestHandleGetLogGroupFields_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) {
-	executor := &cloudWatchExecutor{}
-
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
+	require.NoError(t, err)
+	require.NotNil(t, resp)
+
+	expFrame := &data.Frame{
+		Name: refID,
+		Fields: []*data.Field{
+			data.NewField("name", nil, []*string{
+				aws.String("field_a"), aws.String("field_b"), aws.String("field_c"),
+			}),
+			data.NewField("percent", nil, []*int64{
+				aws.Int64(100), aws.Int64(30), aws.Int64(55),
+			}),
+		},
+		Meta: &data.FrameMeta{
+			PreferredVisualization: "logs",
 		},
 	}
+	expFrame.RefID = refID
+	assert.Equal(t, &tsdb.Response{
+		Results: map[string]*tsdb.QueryResult{
+			refID: {
+				Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
+				RefId:      refID,
+			},
+		},
+	}, resp)
+}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"logGroupName": "group_a",
-		"limit":        50,
+func TestQuery_StartQuery(t *testing.T) {
+	origNewCWLogsClient := newCWLogsClient
+	t.Cleanup(func() {
+		newCWLogsClient = origNewCWLogsClient
 	})
 
-	frame, err := executor.handleGetLogGroupFields(context.Background(), logsClient, params, "A")
-
-	expectedNameField := data.NewField("name", nil, []*string{aws.String("field_a"), aws.String("field_b"), aws.String("field_c")})
-	expectedPercentField := data.NewField("percent", nil, []*int64{aws.Int64(100), aws.Int64(30), aws.Int64(55)})
-	expectedFrame := data.NewFrame("A", expectedNameField, expectedPercentField)
-	expectedFrame.RefID = "A"
-
-	assert.Equal(t, nil, err)
-	assert.Equal(t, expectedFrame, frame)
-}
-
-func TestExecuteStartQuery(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	var cli fakeCWLogsClient
 
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
-		},
+	newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return cli
 	}
 
-	timeRange := &tsdb.TimeRange{
-		From: "1584873443000",
-		To:   "1584700643000",
-	}
+	t.Run("invalid time range", func(t *testing.T) {
+		cli = fakeCWLogsClient{
+			logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
+				LogGroupFields: []*cloudwatchlogs.LogGroupField{
+					{
+						Name:    aws.String("field_a"),
+						Percent: aws.Int64(100),
+					},
+					{
+						Name:    aws.String("field_b"),
+						Percent: aws.Int64(30),
+					},
+					{
+						Name:    aws.String("field_c"),
+						Percent: aws.Int64(55),
+					},
+				},
+			},
+		}
+
+		timeRange := &tsdb.TimeRange{
+			From: "1584873443000",
+			To:   "1584700643000",
+		}
+
+		executor := newExecutor()
+		_, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			TimeRange: timeRange,
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":        "logAction",
+						"subtype":     "StartQuery",
+						"limit":       50,
+						"region":      "default",
+						"queryString": "fields @message",
+					}),
+				},
+			},
+		})
+		require.Error(t, err)
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"region":      "default",
-		"limit":       50,
-		"queryString": "fields @message",
+		assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err)
 	})
 
-	response, err := executor.executeStartQuery(context.Background(), logsClient, params, timeRange)
+	t.Run("valid time range", func(t *testing.T) {
+		const refID = "A"
+		cli = fakeCWLogsClient{
+			logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
+				LogGroupFields: []*cloudwatchlogs.LogGroupField{
+					{
+						Name:    aws.String("field_a"),
+						Percent: aws.Int64(100),
+					},
+					{
+						Name:    aws.String("field_b"),
+						Percent: aws.Int64(30),
+					},
+					{
+						Name:    aws.String("field_c"),
+						Percent: aws.Int64(55),
+					},
+				},
+			},
+		}
+
+		timeRange := &tsdb.TimeRange{
+			From: "1584700643000",
+			To:   "1584873443000",
+		}
+
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			TimeRange: timeRange,
+			Queries: []*tsdb.Query{
+				{
+					RefId: refID,
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":        "logAction",
+						"subtype":     "StartQuery",
+						"limit":       50,
+						"region":      "default",
+						"queryString": "fields @message",
+					}),
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		expFrame := data.NewFrame(
+			refID,
+			data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}),
+		)
+		expFrame.RefID = refID
+		expFrame.Meta = &data.FrameMeta{
+			Custom: map[string]interface{}{
+				"Region": "default",
+			},
+			PreferredVisualization: "logs",
+		}
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				refID: {
+					Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
+					RefId:      refID,
+				},
+			},
+		}, resp)
+	})
+}
 
-	var expectedResponse *cloudwatchlogs.StartQueryOutput = nil
+func TestQuery_StopQuery(t *testing.T) {
+	origNewCWLogsClient := newCWLogsClient
+	t.Cleanup(func() {
+		newCWLogsClient = origNewCWLogsClient
+	})
 
-	assert.Equal(t, expectedResponse, response)
-	assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err)
-}
+	var cli fakeCWLogsClient
 
-func TestHandleStartQuery(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return cli
+	}
 
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
+	cli = fakeCWLogsClient{
+		logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
+			LogGroupFields: []*cloudwatchlogs.LogGroupField{
+				{
+					Name:    aws.String("field_a"),
+					Percent: aws.Int64(100),
+				},
+				{
+					Name:    aws.String("field_b"),
+					Percent: aws.Int64(30),
+				},
+				{
+					Name:    aws.String("field_c"),
+					Percent: aws.Int64(55),
+				},
+			},
 		},
 	}
 
 	timeRange := &tsdb.TimeRange{
-		From: "1584700643000",
-		To:   "1584873443000",
+		From: "1584873443000",
+		To:   "1584700643000",
 	}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"region":      "default",
-		"limit":       50,
-		"queryString": "fields @message",
+	executor := newExecutor()
+	resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+		TimeRange: timeRange,
+		Queries: []*tsdb.Query{
+			{
+				Model: simplejson.NewFromAny(map[string]interface{}{
+					"type":    "logAction",
+					"subtype": "StopQuery",
+					"queryId": "abcd-efgh-ijkl-mnop",
+				}),
+			},
+		},
 	})
+	require.NoError(t, err)
 
-	frame, err := executor.handleStartQuery(context.Background(), logsClient, params, timeRange, "A")
-
-	expectedField := data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"})
-	expectedFrame := data.NewFrame("A", expectedField)
-	expectedFrame.RefID = "A"
-	expectedFrame.Meta = &data.FrameMeta{
-		Custom: map[string]interface{}{
-			"Region": "default",
+	expFrame := &data.Frame{
+		Name: "StopQueryResponse",
+		Fields: []*data.Field{
+			data.NewField("success", nil, []bool{true}),
 		},
-	}
-
-	assert.Equal(t, nil, err)
-	assert.Equal(t, expectedFrame, frame)
-}
-
-func TestHandleStopQuery(t *testing.T) {
-	executor := &cloudWatchExecutor{}
-
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
+		Meta: &data.FrameMeta{
+			PreferredVisualization: "logs",
 		},
 	}
+	assert.Equal(t, &tsdb.Response{
+		Results: map[string]*tsdb.QueryResult{
+			"": {
+				Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
+			},
+		},
+	}, resp)
+}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"queryId": "abcd-efgh-ijkl-mnop",
+func TestQuery_GetQueryResults(t *testing.T) {
+	origNewCWLogsClient := newCWLogsClient
+	t.Cleanup(func() {
+		newCWLogsClient = origNewCWLogsClient
 	})
 
-	frame, err := executor.handleStopQuery(context.Background(), logsClient, params)
-
-	expectedField := data.NewField("success", nil, []bool{true})
-	expectedFrame := data.NewFrame("StopQueryResponse", expectedField)
-
-	assert.Equal(t, nil, err)
-	assert.Equal(t, expectedFrame, frame)
-}
+	var cli fakeCWLogsClient
 
-func TestHandleGetQueryResults(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return cli
+	}
 
-	logsClient := &FakeLogsClient{
-		Config: aws.Config{
-			Region: aws.String("default"),
+	const refID = "A"
+	cli = fakeCWLogsClient{
+		queryResults: cloudwatchlogs.GetQueryResultsOutput{
+			Results: [][]*cloudwatchlogs.ResultField{
+				{
+					{
+						Field: aws.String("@timestamp"),
+						Value: aws.String("2020-03-20 10:37:23.000"),
+					},
+					{
+						Field: aws.String("field_b"),
+						Value: aws.String("b_1"),
+					},
+					{
+						Field: aws.String("@ptr"),
+						Value: aws.String("abcdefg"),
+					},
+				},
+				{
+					{
+						Field: aws.String("@timestamp"),
+						Value: aws.String("2020-03-20 10:40:43.000"),
+					},
+					{
+						Field: aws.String("field_b"),
+						Value: aws.String("b_2"),
+					},
+					{
+						Field: aws.String("@ptr"),
+						Value: aws.String("hijklmnop"),
+					},
+				},
+			},
+			Statistics: &cloudwatchlogs.QueryStatistics{
+				BytesScanned:   aws.Float64(512),
+				RecordsMatched: aws.Float64(256),
+				RecordsScanned: aws.Float64(1024),
+			},
+			Status: aws.String("Complete"),
 		},
 	}
 
-	params := simplejson.NewFromAny(map[string]interface{}{
-		"queryId": "abcd-efgh-ijkl-mnop",
+	executor := newExecutor()
+	resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+		Queries: []*tsdb.Query{
+			{
+				RefId: refID,
+				Model: simplejson.NewFromAny(map[string]interface{}{
+					"type":    "logAction",
+					"subtype": "GetQueryResults",
+					"queryId": "abcd-efgh-ijkl-mnop",
+				}),
+			},
+		},
 	})
-
-	frame, err := executor.handleGetQueryResults(context.Background(), logsClient, params, "A")
 	require.NoError(t, err)
-	timeA, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
+
+	time1, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
 	require.NoError(t, err)
-	timeB, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000")
+	time2, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000")
 	require.NoError(t, err)
-	expectedTimeField := data.NewField("@timestamp", nil, []*time.Time{
-		aws.Time(timeA), aws.Time(timeB),
+	expField1 := data.NewField("@timestamp", nil, []*time.Time{
+		aws.Time(time1), aws.Time(time2),
 	})
-	expectedTimeField.SetConfig(&data.FieldConfig{DisplayName: "Time"})
-
-	expectedFieldB := data.NewField("field_b", nil, []*string{
+	expField1.SetConfig(&data.FieldConfig{DisplayName: "Time"})
+	expField2 := data.NewField("field_b", nil, []*string{
 		aws.String("b_1"), aws.String("b_2"),
 	})
-
-	expectedFrame := data.NewFrame("A", expectedTimeField, expectedFieldB)
-	expectedFrame.RefID = "A"
-
-	expectedFrame.Meta = &data.FrameMeta{
+	expFrame := data.NewFrame(refID, expField1, expField2)
+	expFrame.RefID = refID
+	expFrame.Meta = &data.FrameMeta{
 		Custom: map[string]interface{}{
 			"Status": "Complete",
 			"Statistics": cloudwatchlogs.QueryStatistics{
@@ -213,9 +495,15 @@ func TestHandleGetQueryResults(t *testing.T) {
 				RecordsScanned: aws.Float64(1024),
 			},
 		},
+		PreferredVisualization: "logs",
 	}
 
-	assert.Equal(t, nil, err)
-	assert.ElementsMatch(t, expectedFrame.Fields, frame.Fields)
-	assert.Equal(t, expectedFrame.Meta, frame.Meta)
+	assert.Equal(t, &tsdb.Response{
+		Results: map[string]*tsdb.QueryResult{
+			refID: {
+				RefId:      refID,
+				Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
+			},
+		},
+	}, resp)
 }

pkg/tsdb/cloudwatch/logs_client_mock.go

@@ -1,106 +0,0 @@
-package cloudwatch
-
-import (
-	"context"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/request"
-	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
-	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
-)
-
-type FakeLogsClient struct {
-	cloudwatchlogsiface.CloudWatchLogsAPI
-	Config aws.Config
-}
-
-func (f FakeLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
-	return &cloudwatchlogs.DescribeLogGroupsOutput{
-		LogGroups: []*cloudwatchlogs.LogGroup{
-			{
-				LogGroupName: aws.String("group_a"),
-			},
-			{
-				LogGroupName: aws.String("group_b"),
-			},
-			{
-				LogGroupName: aws.String("group_c"),
-			},
-		},
-	}, nil
-}
-
-func (f FakeLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
-	return &cloudwatchlogs.GetLogGroupFieldsOutput{
-		LogGroupFields: []*cloudwatchlogs.LogGroupField{
-			{
-				Name:    aws.String("field_a"),
-				Percent: aws.Int64(100),
-			},
-			{
-				Name:    aws.String("field_b"),
-				Percent: aws.Int64(30),
-			},
-			{
-				Name:    aws.String("field_c"),
-				Percent: aws.Int64(55),
-			},
-		},
-	}, nil
-}
-
-func (f FakeLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
-	return &cloudwatchlogs.StartQueryOutput{
-		QueryId: aws.String("abcd-efgh-ijkl-mnop"),
-	}, nil
-}
-
-func (f FakeLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
-	return &cloudwatchlogs.StopQueryOutput{
-		Success: aws.Bool(true),
-	}, nil
-}
-
-func (f FakeLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
-	return &cloudwatchlogs.GetQueryResultsOutput{
-		Results: [][]*cloudwatchlogs.ResultField{
-			{
-				{
-					Field: aws.String("@timestamp"),
-					Value: aws.String("2020-03-20 10:37:23.000"),
-				},
-				{
-					Field: aws.String("field_b"),
-					Value: aws.String("b_1"),
-				},
-				{
-					Field: aws.String("@ptr"),
-					Value: aws.String("abcdefg"),
-				},
-			},
-
-			{
-				{
-					Field: aws.String("@timestamp"),
-					Value: aws.String("2020-03-20 10:40:43.000"),
-				},
-				{
-					Field: aws.String("field_b"),
-					Value: aws.String("b_2"),
-				},
-				{
-					Field: aws.String("@ptr"),
-					Value: aws.String("hijklmnop"),
-				},
-			},
-		},
-
-		Statistics: &cloudwatchlogs.QueryStatistics{
-			BytesScanned:   aws.Float64(512),
-			RecordsMatched: aws.Float64(256),
-			RecordsScanned: aws.Float64(1024),
-		},
-
-		Status: aws.String("Complete"),
-	}, nil
-}

pkg/tsdb/cloudwatch/metric_find_query.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awsutil"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
@@ -310,8 +309,7 @@ func parseMultiSelectValue(input string) []string {
 // Whenever this list is updated, the frontend list should also be updated.
 // Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
 func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
-	const region = "default"
-	dsInfo := e.getDSInfo(region)
+	dsInfo := e.getDSInfo(defaultRegion)
 	profile := dsInfo.Profile
 	if cache, ok := regionCache.Load(profile); ok {
 		if cache2, ok2 := cache.([]suggestData); ok2 {
@@ -319,12 +317,12 @@ func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s
 		}
 	}
 
-	err := e.ensureClientSession("default")
+	client, err := e.getEC2Client(defaultRegion)
 	if err != nil {
 		return nil, err
 	}
 	regions := knownRegions
-	r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{})
+	r, err := client.DescribeRegions(&ec2.DescribeRegionsInput{})
 	if err != nil {
 		// ignore error for backward compatibility
 		plog.Error("Failed to get regions", "error", err)
@@ -389,7 +387,7 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s
 		dsInfo := e.getDSInfo(region)
 		dsInfo.Namespace = namespace
 
-		if namespaceMetrics, err = e.getMetricsForCustomMetrics(region, e.getAllMetrics); err != nil {
+		if namespaceMetrics, err = e.getMetricsForCustomMetrics(region); err != nil {
 			return nil, errors.New("Unable to call AWS API")
 		}
 	}
@@ -418,7 +416,7 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters
 		dsInfo := e.getDSInfo(region)
 		dsInfo.Namespace = namespace
 
-		if dimensionValues, err = e.getDimensionsForCustomMetrics(region, e.getAllMetrics); err != nil {
+		if dimensionValues, err = e.getDimensionsForCustomMetrics(region); err != nil {
 			return nil, errors.New("Unable to call AWS API")
 		}
 	}
@@ -483,31 +481,10 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
 	return result, nil
 }
 
-func (e *cloudWatchExecutor) ensureClientSession(region string) error {
-	if e.ec2Svc == nil {
-		dsInfo := e.getDSInfo(region)
-		cfg, err := getAwsConfig(dsInfo)
-		if err != nil {
-			return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
-		}
-		sess, err := session.NewSession(cfg)
-		if err != nil {
-			return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
-		}
-		e.ec2Svc = ec2.New(sess, cfg)
-	}
-	return nil
-}
-
 func (e *cloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
 	region := parameters.Get("region").MustString()
 	instanceId := parameters.Get("instanceId").MustString()
 
-	err := e.ensureClientSession(region)
-	if err != nil {
-		return nil, err
-	}
-
 	instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId))
 	instances, err := e.ec2DescribeInstances(region, nil, instanceIds)
 	if err != nil {
@@ -547,11 +524,6 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
 		}
 	}
 
-	err := e.ensureClientSession(region)
-	if err != nil {
-		return nil, err
-	}
-
 	instances, err := e.ec2DescribeInstances(region, filters, nil)
 	if err != nil {
 		return nil, err
@@ -610,32 +582,11 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
 	return result, nil
 }
 
-func (e *cloudWatchExecutor) ensureRGTAClientSession(region string) error {
-	if e.rgtaSvc == nil {
-		dsInfo := e.getDSInfo(region)
-		cfg, err := getAwsConfig(dsInfo)
-		if err != nil {
-			return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
-		}
-		sess, err := session.NewSession(cfg)
-		if err != nil {
-			return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
-		}
-		e.rgtaSvc = resourcegroupstaggingapi.New(sess, cfg)
-	}
-	return nil
-}
-
 func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
 	region := parameters.Get("region").MustString()
 	resourceType := parameters.Get("resourceType").MustString()
 	filterJson := parameters.Get("tags").MustMap()
 
-	err := e.ensureRGTAClientSession(region)
-	if err != nil {
-		return nil, err
-	}
-
 	var filters []*resourcegroupstaggingapi.TagFilter
 	for k, v := range filterJson {
 		if vv, ok := v.([]interface{}); ok {
@@ -706,8 +657,13 @@ func (e *cloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.
 		InstanceIds: instanceIds,
 	}
 
+	client, err := e.getEC2Client(region)
+	if err != nil {
+		return nil, err
+	}
+
 	var resp ec2.DescribeInstancesOutput
-	if err := e.ec2Svc.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
+	if err := client.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
 		resp.Reservations = append(resp.Reservations, page.Reservations...)
 		return !lastPage
 	}); err != nil {
@@ -724,8 +680,13 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
 		TagFilters:          filters,
 	}
 
+	client, err := e.getRGTAClient(region)
+	if err != nil {
+		return nil, err
+	}
+
 	var resp resourcegroupstaggingapi.GetResourcesOutput
-	if err := e.rgtaSvc.GetResourcesPages(params,
+	if err := client.GetResourcesPages(params,
 		func(page *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool {
 			resp.ResourceTagMappingList = append(resp.ResourceTagMappingList, page.ResourceTagMappingList...)
 			return !lastPage
@@ -737,27 +698,18 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
 }
 
 func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetricsOutput, error) {
-	dsInfo := e.getDSInfo(region)
-	creds, err := getCredentials(dsInfo)
+	client, err := e.getCWClient(region)
 	if err != nil {
 		return cloudwatch.ListMetricsOutput{}, err
 	}
-	cfg := &aws.Config{
-		Region:      aws.String(dsInfo.Region),
-		Credentials: creds,
-	}
-	sess, err := session.NewSession(cfg)
-	if err != nil {
-		return cloudwatch.ListMetricsOutput{}, err
-	}
-	svc := cloudwatch.New(sess, cfg)
 
+	dsInfo := e.getDSInfo(region)
 	params := &cloudwatch.ListMetricsInput{
 		Namespace: aws.String(dsInfo.Namespace),
 	}
 
 	var resp cloudwatch.ListMetricsOutput
-	err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
+	err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
 		metrics.MAwsCloudWatchListMetrics.Inc()
 		metrics, err := awsutil.ValuesAtPath(page, "Metrics")
 		if err != nil {
@@ -769,12 +721,13 @@ func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetric
 		}
 		return !lastPage
 	})
+
 	return resp, err
 }
 
 var metricsCacheLock sync.Mutex
 
-func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
+func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string) ([]string, error) {
 	metricsCacheLock.Lock()
 	defer metricsCacheLock.Unlock()
 
@@ -794,7 +747,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
 	if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
 		return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
 	}
-	result, err := getAllMetrics(region)
+	result, err := e.getAllMetrics(region)
 	if err != nil {
 		return []string{}, err
 	}
@@ -813,7 +766,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
 
 var dimensionsCacheLock sync.Mutex
 
-func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
+func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string) ([]string, error) {
 	dimensionsCacheLock.Lock()
 	defer dimensionsCacheLock.Unlock()
 
@@ -833,7 +786,7 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAll
 	if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
 		return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
 	}
-	result, err := getAllMetrics(region)
+	result, err := e.getAllMetrics(region)
 	if err != nil {
 		return []string{}, err
 	}

pkg/tsdb/cloudwatch/metric_find_query_test.go

@@ -5,278 +5,497 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
-	"github.com/grafana/grafana/pkg/components/securejsondata"
 	"github.com/grafana/grafana/pkg/components/simplejson"
-	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-type mockedEc2 struct {
-	ec2iface.EC2API
-	Resp        ec2.DescribeInstancesOutput
-	RespRegions ec2.DescribeRegionsOutput
-}
+func TestQuery_Metrics(t *testing.T) {
+	origNewCWClient := newCWClient
+	t.Cleanup(func() {
+		newCWClient = origNewCWClient
+	})
 
-type mockedRGTA struct {
-	resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
-	Resp resourcegroupstaggingapi.GetResourcesOutput
-}
+	var client fakeCWClient
 
-func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
-	fn(&m.Resp, true)
-	return nil
-}
-func (m mockedEc2) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
-	return &m.RespRegions, nil
-}
+	newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
+		return client
+	}
 
-func (m mockedRGTA) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput, fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error {
-	fn(&m.Resp, true)
-	return nil
-}
-
-func TestCloudWatchMetrics(t *testing.T) {
-	t.Run("When calling getMetricsForCustomMetrics", func(t *testing.T) {
-		const region = "us-east-1"
-		e := &cloudWatchExecutor{
-			DataSource: &models.DataSource{
-				Database: "default",
-				JsonData: simplejson.NewFromAny(map[string]interface{}{
-					"Region": region,
-				}),
+	t.Run("Custom metrics", func(t *testing.T) {
+		client = fakeCWClient{
+			metrics: []*cloudwatch.Metric{
+				{
+					MetricName: aws.String("Test_MetricName"),
+					Dimensions: []*cloudwatch.Dimension{
+						{
+							Name: aws.String("Test_DimensionName"),
+						},
+					},
+				},
 			},
 		}
-		f := func(region string) (cloudwatch.ListMetricsOutput, error) {
-			return cloudwatch.ListMetricsOutput{
-				Metrics: []*cloudwatch.Metric{
-					{
-						MetricName: aws.String("Test_MetricName"),
-						Dimensions: []*cloudwatch.Dimension{
-							{
-								Name: aws.String("Test_DimensionName"),
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":      "metricFindQuery",
+						"subtype":   "metrics",
+						"region":    "us-east-1",
+						"namespace": "custom",
+					}),
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": 1,
+					}),
+					Tables: []*tsdb.Table{
+						{
+							Columns: []tsdb.TableColumn{
+								{
+									Text: "text",
+								},
+								{
+									Text: "value",
+								},
+							},
+							Rows: []tsdb.RowValues{
+								{
+									"Test_MetricName",
+									"Test_MetricName",
+								},
 							},
 						},
 					},
 				},
-			}, nil
+			},
+		}, resp)
+	})
+
+	t.Run("Dimension keys for custom metrics", func(t *testing.T) {
+		client = fakeCWClient{
+			metrics: []*cloudwatch.Metric{
+				{
+					MetricName: aws.String("Test_MetricName"),
+					Dimensions: []*cloudwatch.Dimension{
+						{
+							Name: aws.String("Test_DimensionName"),
+						},
+					},
+				},
+			},
 		}
-		metrics, err := e.getMetricsForCustomMetrics(region, f)
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":      "metricFindQuery",
+						"subtype":   "dimension_keys",
+						"region":    "us-east-1",
+						"namespace": "custom",
+					}),
+				},
+			},
+		})
 		require.NoError(t, err)
 
-		assert.Contains(t, metrics, "Test_MetricName")
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": 1,
+					}),
+					Tables: []*tsdb.Table{
+						{
+							Columns: []tsdb.TableColumn{
+								{
+									Text: "text",
+								},
+								{
+									Text: "value",
+								},
+							},
+							Rows: []tsdb.RowValues{
+								{
+									"Test_DimensionName",
+									"Test_DimensionName",
+								},
+							},
+						},
+					},
+				},
+			},
+		}, resp)
+	})
+}
+
+func TestQuery_Regions(t *testing.T) {
+	origNewEC2Client := newEC2Client
+	t.Cleanup(func() {
+		newEC2Client = origNewEC2Client
 	})
 
-	t.Run("When calling getDimensionsForCustomMetrics", func(t *testing.T) {
-		const region = "us-east-1"
-		e := &cloudWatchExecutor{
-			DataSource: &models.DataSource{
-				Database: "default",
-				JsonData: simplejson.NewFromAny(map[string]interface{}{
-					"Region": region,
-				}),
+	var cli fakeEC2Client
+
+	newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
+		return cli
+	}
+
+	t.Run("An extra region", func(t *testing.T) {
+		const regionName = "xtra-region"
+		cli = fakeEC2Client{
+			regions: []string{regionName},
+		}
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":      "metricFindQuery",
+						"subtype":   "regions",
+						"region":    "us-east-1",
+						"namespace": "custom",
+					}),
+				},
 			},
+		})
+		require.NoError(t, err)
+
+		rows := []tsdb.RowValues{}
+		for _, region := range knownRegions {
+			rows = append(rows, []interface{}{
+				region,
+				region,
+			})
 		}
-		f := func(region string) (cloudwatch.ListMetricsOutput, error) {
-			return cloudwatch.ListMetricsOutput{
-				Metrics: []*cloudwatch.Metric{
-					{
-						MetricName: aws.String("Test_MetricName"),
-						Dimensions: []*cloudwatch.Dimension{
-							{
-								Name: aws.String("Test_DimensionName"),
+		rows = append(rows, []interface{}{
+			regionName,
+			regionName,
+		})
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": len(knownRegions) + 1,
+					}),
+					Tables: []*tsdb.Table{
+						{
+							Columns: []tsdb.TableColumn{
+								{
+									Text: "text",
+								},
+								{
+									Text: "value",
+								},
 							},
+							Rows: rows,
 						},
 					},
 				},
-			}, nil
-		}
-		dimensionKeys, err := e.getDimensionsForCustomMetrics(region, f)
-		require.NoError(t, err)
+			},
+		}, resp)
+	})
+}
 
-		assert.Contains(t, dimensionKeys, "Test_DimensionName")
+func TestQuery_InstanceAttributes(t *testing.T) {
+	origNewEC2Client := newEC2Client
+	t.Cleanup(func() {
+		newEC2Client = origNewEC2Client
 	})
 
-	t.Run("When calling handleGetRegions", func(t *testing.T) {
-		executor := &cloudWatchExecutor{
-			ec2Svc: mockedEc2{RespRegions: ec2.DescribeRegionsOutput{
-				Regions: []*ec2.Region{
-					{
-						RegionName: aws.String("ap-northeast-2"),
+	var cli fakeEC2Client
+
+	newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
+		return cli
+	}
+
+	t.Run("Get instance ID", func(t *testing.T) {
+		const instanceID = "i-12345678"
+		cli = fakeEC2Client{
+			reservations: []*ec2.Reservation{
+				{
+					Instances: []*ec2.Instance{
+						{
+							InstanceId: aws.String(instanceID),
+							Tags: []*ec2.Tag{
+								{
+									Key:   aws.String("Environment"),
+									Value: aws.String("production"),
+								},
+							},
+						},
 					},
 				},
-			}},
-		}
-		jsonData := simplejson.New()
-		jsonData.Set("defaultRegion", "default")
-		executor.DataSource = &models.DataSource{
-			JsonData:       jsonData,
-			SecureJsonData: securejsondata.SecureJsonData{},
+			},
 		}
-
-		result, err := executor.handleGetRegions(context.Background(), simplejson.New(), &tsdb.TsdbQuery{})
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":          "metricFindQuery",
+						"subtype":       "ec2_instance_attribute",
+						"region":        "us-east-1",
+						"attributeName": "InstanceId",
+						"filters": map[string]interface{}{
+							"tag:Environment": []string{"production"},
+						},
+					}),
+				},
+			},
+		})
 		require.NoError(t, err)
 
-		assert.Equal(t, "af-south-1", result[0].Text)
-		assert.Equal(t, "ap-east-1", result[1].Text)
-		assert.Equal(t, "ap-northeast-1", result[2].Text)
-		assert.Equal(t, "ap-northeast-2", result[3].Text)
-	})
-
-	t.Run("When calling handleGetEc2InstanceAttribute", func(t *testing.T) {
-		executor := &cloudWatchExecutor{
-			ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{
-				Reservations: []*ec2.Reservation{
-					{
-						Instances: []*ec2.Instance{
-							{
-								InstanceId: aws.String("i-12345678"),
-								Tags: []*ec2.Tag{
-									{
-										Key:   aws.String("Environment"),
-										Value: aws.String("production"),
-									},
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": 1,
+					}),
+					Tables: []*tsdb.Table{
+						{
+							Columns: []tsdb.TableColumn{
+								{
+									Text: "text",
+								},
+								{
+									Text: "value",
+								},
+							},
+							Rows: []tsdb.RowValues{
+								{
+									instanceID,
+									instanceID,
 								},
 							},
 						},
 					},
 				},
-			}},
-		}
-
-		json := simplejson.New()
-		json.Set("region", "us-east-1")
-		json.Set("attributeName", "InstanceId")
-		filters := make(map[string]interface{})
-		filters["tag:Environment"] = []string{"production"}
-		json.Set("filters", filters)
-		result, err := executor.handleGetEc2InstanceAttribute(context.Background(), json, &tsdb.TsdbQuery{})
-		require.NoError(t, err)
+			},
+		}, resp)
+	})
+}
 
-		assert.Equal(t, "i-12345678", result[0].Text)
+func TestQuery_EBSVolumeIDs(t *testing.T) {
+	origNewEC2Client := newEC2Client
+	t.Cleanup(func() {
+		newEC2Client = origNewEC2Client
 	})
 
-	t.Run("When calling handleGetEbsVolumeIds", func(t *testing.T) {
-		executor := &cloudWatchExecutor{
-			ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{
-				Reservations: []*ec2.Reservation{
-					{
-						Instances: []*ec2.Instance{
-							{
-								InstanceId: aws.String("i-1"),
-								BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}},
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}},
-								},
+	var cli fakeEC2Client
+
+	newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
+		return cli
+	}
+
+	t.Run("", func(t *testing.T) {
+		const instanceIDs = "{i-1, i-2, i-3}"
+
+		cli = fakeEC2Client{
+			reservations: []*ec2.Reservation{
+				{
+					Instances: []*ec2.Instance{
+						{
+							InstanceId: aws.String("i-1"),
+							BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}},
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}},
 							},
-							{
-								InstanceId: aws.String("i-2"),
-								BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}},
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}},
-								},
+						},
+						{
+							InstanceId: aws.String("i-2"),
+							BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}},
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}},
 							},
 						},
 					},
-					{
-						Instances: []*ec2.Instance{
-							{
-								InstanceId: aws.String("i-3"),
-								BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}},
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}},
-								},
+				},
+				{
+					Instances: []*ec2.Instance{
+						{
+							InstanceId: aws.String("i-3"),
+							BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}},
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}},
 							},
-							{
-								InstanceId: aws.String("i-4"),
-								BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}},
-									{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}},
-								},
+						},
+						{
+							InstanceId: aws.String("i-4"),
+							BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}},
+								{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}},
 							},
 						},
 					},
 				},
-			}},
+			},
 		}
-
-		json := simplejson.New()
-		json.Set("region", "us-east-1")
-		json.Set("instanceId", "{i-1, i-2, i-3, i-4}")
-		result, err := executor.handleGetEbsVolumeIds(context.Background(), json, &tsdb.TsdbQuery{})
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":       "metricFindQuery",
+						"subtype":    "ebs_volume_ids",
+						"region":     "us-east-1",
+						"instanceId": instanceIDs,
+					}),
+				},
+			},
+		})
 		require.NoError(t, err)
 
-		require.Len(t, result, 8)
-		assert.Equal(t, "vol-1-1", result[0].Text)
-		assert.Equal(t, "vol-1-2", result[1].Text)
-		assert.Equal(t, "vol-2-1", result[2].Text)
-		assert.Equal(t, "vol-2-2", result[3].Text)
-		assert.Equal(t, "vol-3-1", result[4].Text)
-		assert.Equal(t, "vol-3-2", result[5].Text)
-		assert.Equal(t, "vol-4-1", result[6].Text)
-		assert.Equal(t, "vol-4-2", result[7].Text)
-	})
-
-	t.Run("When calling handleGetResourceArns", func(t *testing.T) {
-		executor := &cloudWatchExecutor{
-			rgtaSvc: mockedRGTA{
-				Resp: resourcegroupstaggingapi.GetResourcesOutput{
-					ResourceTagMappingList: []*resourcegroupstaggingapi.ResourceTagMapping{
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": 6,
+					}),
+					Tables: []*tsdb.Table{
 						{
-							ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"),
-							Tags: []*resourcegroupstaggingapi.Tag{
+							Columns: []tsdb.TableColumn{
 								{
-									Key:   aws.String("Environment"),
-									Value: aws.String("production"),
+									Text: "text",
+								},
+								{
+									Text: "value",
 								},
 							},
-						},
-						{
-							ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"),
-							Tags: []*resourcegroupstaggingapi.Tag{
+							Rows: []tsdb.RowValues{
 								{
-									Key:   aws.String("Environment"),
-									Value: aws.String("production"),
+									"vol-1-1",
+									"vol-1-1",
+								},
+								{
+									"vol-1-2",
+									"vol-1-2",
+								},
+								{
+									"vol-2-1",
+									"vol-2-1",
+								},
+								{
+									"vol-2-2",
+									"vol-2-2",
+								},
+								{
+									"vol-3-1",
+									"vol-3-1",
+								},
+								{
+									"vol-3-2",
+									"vol-3-2",
 								},
 							},
 						},
 					},
 				},
 			},
-		}
-
-		json := simplejson.New()
-		json.Set("region", "us-east-1")
-		json.Set("resourceType", "ec2:instance")
-		tags := make(map[string]interface{})
-		tags["Environment"] = []string{"production"}
-		json.Set("tags", tags)
-		result, err := executor.handleGetResourceArns(context.Background(), json, &tsdb.TsdbQuery{})
-		require.NoError(t, err)
-
-		assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Text)
-		assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Value)
-		assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Text)
-		assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Value)
+		}, resp)
 	})
 }
 
-func TestParseMultiSelectValue(t *testing.T) {
-	values := parseMultiSelectValue(" i-someInstance ")
-	assert.Equal(t, []string{"i-someInstance"}, values)
+func TestQuery_ResourceARNs(t *testing.T) {
+	origNewRGTAClient := newRGTAClient
+	t.Cleanup(func() {
+		newRGTAClient = origNewRGTAClient
+	})
 
-	values = parseMultiSelectValue("{i-05}")
-	assert.Equal(t, []string{"i-05"}, values)
+	var cli fakeRGTAClient
 
-	values = parseMultiSelectValue(" {i-01, i-03, i-04} ")
-	assert.Equal(t, []string{"i-01", "i-03", "i-04"}, values)
+	newRGTAClient = func(client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
+		return cli
+	}
+
+	t.Run("", func(t *testing.T) {
+		cli = fakeRGTAClient{
+			tagMapping: []*resourcegroupstaggingapi.ResourceTagMapping{
+				{
+					ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"),
+					Tags: []*resourcegroupstaggingapi.Tag{
+						{
+							Key:   aws.String("Environment"),
+							Value: aws.String("production"),
+						},
+					},
+				},
+				{
+					ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"),
+					Tags: []*resourcegroupstaggingapi.Tag{
+						{
+							Key:   aws.String("Environment"),
+							Value: aws.String("production"),
+						},
+					},
+				},
+			},
+		}
+		executor := newExecutor()
+		resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
+			Queries: []*tsdb.Query{
+				{
+					Model: simplejson.NewFromAny(map[string]interface{}{
+						"type":         "metricFindQuery",
+						"subtype":      "resource_arns",
+						"region":       "us-east-1",
+						"resourceType": "ec2:instance",
+						"tags": map[string]interface{}{
+							"Environment": []string{"production"},
+						},
+					}),
+				},
+			},
+		})
+		require.NoError(t, err)
 
-	values = parseMultiSelectValue("i-{01}")
-	assert.Equal(t, []string{"i-{01}"}, values)
+		assert.Equal(t, &tsdb.Response{
+			Results: map[string]*tsdb.QueryResult{
+				"": {
+					Meta: simplejson.NewFromAny(map[string]interface{}{
+						"rowCount": 2,
+					}),
+					Tables: []*tsdb.Table{
+						{
+							Columns: []tsdb.TableColumn{
+								{
+									Text: "text",
+								},
+								{
+									Text: "value",
+								},
+							},
+							Rows: []tsdb.RowValues{
+								{
+									"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
+									"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
+								},
+								{
+									"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
+									"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
+								},
+							},
+						},
+					},
+				},
+			},
+		}, resp)
+	})
 }

pkg/tsdb/cloudwatch/query_transformer_test.go

@@ -10,7 +10,7 @@ import (
 func TestQueryTransformer(t *testing.T) {
 	Convey("TestQueryTransformer", t, func() {
 		Convey("when transforming queries", func() {
-			executor := &cloudWatchExecutor{}
+			executor := newExecutor()
 			Convey("one cloudwatchQuery is generated when its request query has one stat", func() {
 				requestQueries := []*requestQuery{
 					{

pkg/tsdb/cloudwatch/test_utils.go

@@ -0,0 +1,133 @@
+package cloudwatch
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
+	"github.com/grafana/grafana/pkg/components/securejsondata"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+func fakeDataSource() *models.DataSource {
+	jsonData := simplejson.New()
+	jsonData.Set("defaultRegion", "default")
+	return &models.DataSource{
+		Id:             1,
+		Database:       "default",
+		JsonData:       jsonData,
+		SecureJsonData: securejsondata.SecureJsonData{},
+	}
+}
+
+type fakeCWLogsClient struct {
+	cloudwatchlogsiface.CloudWatchLogsAPI
+	logGroups      cloudwatchlogs.DescribeLogGroupsOutput
+	logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput
+	queryResults   cloudwatchlogs.GetQueryResultsOutput
+}
+
+func (m fakeCWLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
+	return &m.queryResults, nil
+}
+
+func (m fakeCWLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
+	return &cloudwatchlogs.StartQueryOutput{
+		QueryId: aws.String("abcd-efgh-ijkl-mnop"),
+	}, nil
+}
+
+func (m fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
+	return &cloudwatchlogs.StopQueryOutput{
+		Success: aws.Bool(true),
+	}, nil
+}
+
+func (m fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
+	return &m.logGroups, nil
+}
+
+func (m fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
+	return &m.logGroupFields, nil
+}
+
+type fakeCWClient struct {
+	cloudwatchiface.CloudWatchAPI
+
+	metrics []*cloudwatch.Metric
+}
+
+func (c fakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
+	fn(&cloudwatch.ListMetricsOutput{
+		Metrics: c.metrics,
+	}, true)
+	return nil
+}
+
+type fakeEC2Client struct {
+	ec2iface.EC2API
+
+	regions      []string
+	reservations []*ec2.Reservation
+}
+
+func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
+	regions := []*ec2.Region{}
+	for _, region := range c.regions {
+		regions = append(regions, &ec2.Region{
+			RegionName: aws.String(region),
+		})
+	}
+	return &ec2.DescribeRegionsOutput{
+		Regions: regions,
+	}, nil
+}
+
+func (c fakeEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput,
+	fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
+	reservations := []*ec2.Reservation{}
+	for _, r := range c.reservations {
+		instances := []*ec2.Instance{}
+		for _, inst := range r.Instances {
+			if len(in.InstanceIds) == 0 {
+				instances = append(instances, inst)
+				continue
+			}
+
+			for _, id := range in.InstanceIds {
+				if *inst.InstanceId == *id {
+					instances = append(instances, inst)
+				}
+			}
+		}
+		reservation := &ec2.Reservation{Instances: instances}
+		reservations = append(reservations, reservation)
+	}
+	fn(&ec2.DescribeInstancesOutput{
+		Reservations: reservations,
+	}, true)
+	return nil
+}
+
+type fakeRGTAClient struct {
+	resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
+
+	tagMapping []*resourcegroupstaggingapi.ResourceTagMapping
+}
+
+func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput,
+	fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error {
+	fn(&resourcegroupstaggingapi.GetResourcesOutput{
+		ResourceTagMappingList: c.tagMapping,
+	}, true)
+	return nil
+}

pkg/tsdb/cloudwatch/time_series_query_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestTimeSeriesQuery(t *testing.T) {
-	executor := &cloudWatchExecutor{}
+	executor := newExecutor()
 
 	t.Run("End time before start time should result in error", func(t *testing.T) {
 		_, err := executor.executeTimeSeriesQuery(context.TODO(), &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")})