From 00bef917eea202ac7583423441fb1dc9923e65de Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 14 Nov 2019 10:59:41 +0100 Subject: [PATCH] CloudWatch: Datasource improvements (#20268) * CloudWatch: Datasource improvements * Add statistic as template variale * Add wildcard to list of values * Template variable intercept dimension key * Return row specific errors when transformation error occured * Add meta feedback * Make it possible to retrieve values without known metrics * Add curated dashboard for EC2 * Fix broken tests * Use correct dashboard name * Display alert in case multi template var is being used for some certain props in the cloudwatch query * Minor fixes after feedback * Update dashboard json * Update snapshot test * Make sure region default is intercepted in cloudwatch link * Update dashboards * Include ec2 dashboard in ds * Do not include ec2 dashboard in beta1 * Display actual region --- pkg/tsdb/cloudwatch/annotation_query.go | 34 +- pkg/tsdb/cloudwatch/cloudwatch.go | 176 +-- pkg/tsdb/cloudwatch/cloudwatch_query.go | 62 + pkg/tsdb/cloudwatch/cloudwatch_query_test.go | 175 +++ pkg/tsdb/cloudwatch/cloudwatch_test.go | 35 - pkg/tsdb/cloudwatch/constants.go | 30 - pkg/tsdb/cloudwatch/credentials.go | 8 + pkg/tsdb/cloudwatch/get_metric_data.go | 171 --- .../cloudwatch/get_metric_data_executor.go | 34 + .../get_metric_data_executor_test.go | 50 + pkg/tsdb/cloudwatch/get_metric_data_test.go | 116 -- pkg/tsdb/cloudwatch/get_metric_statistics.go | 276 ---- .../cloudwatch/get_metric_statistics_test.go | 187 --- .../cloudwatch/metric_data_input_builder.go | 46 + .../metric_data_input_builder_test.go | 28 + .../cloudwatch/metric_data_query_builder.go | 139 ++ .../metric_data_query_builder_test.go | 215 +++ pkg/tsdb/cloudwatch/metric_find_query.go | 5 +- pkg/tsdb/cloudwatch/query_transformer.go | 103 ++ pkg/tsdb/cloudwatch/query_transformer_test.go | 167 +++ pkg/tsdb/cloudwatch/request_parser.go | 162 +++ pkg/tsdb/cloudwatch/request_parser_test.go | 87 ++ pkg/tsdb/cloudwatch/response_parser.go | 155 +++ pkg/tsdb/cloudwatch/response_parser_test.go | 61 + pkg/tsdb/cloudwatch/time_series_query.go | 101 ++ pkg/tsdb/cloudwatch/types.go | 38 +- .../AppNotifications/AppNotificationItem.tsx | 2 +- public/app/core/copy/appNotification.ts | 5 +- .../cloudwatch/components/Alias.test.tsx | 10 + .../cloudwatch/components/Alias.tsx | 21 + .../components/ConfigEditor.test.tsx | 106 ++ .../cloudwatch/components/ConfigEditor.tsx | 390 ++++++ .../cloudwatch/components/Dimensions.test.tsx | 50 + .../cloudwatch/components/Dimensions.tsx | 80 ++ .../cloudwatch/components/Forms.tsx | 28 + .../components/QueryEditor.test.tsx | 102 ++ .../cloudwatch/components/QueryEditor.tsx | 277 ++++ .../cloudwatch/components/Stats.test.tsx | 21 + .../cloudwatch/components/Stats.tsx | 47 + .../components/ThrottlingErrorMessage.tsx | 27 + .../__snapshots__/Alias.test.tsx.snap | 18 + .../__snapshots__/ConfigEditor.test.tsx.snap | 901 ++++++++++++ .../__snapshots__/QueryEditor.test.tsx.snap | 396 ++++++ .../__snapshots__/Stats.test.tsx.snap | 38 + .../datasource/cloudwatch/components/index.ts | 4 + .../datasource/cloudwatch/config_ctrl.ts | 89 -- .../datasource/cloudwatch/dashboards/EBS.json | 1024 ++++++++++++++ .../cloudwatch/dashboards/Lambda.json | 545 ++++++++ .../datasource/cloudwatch/dashboards/ec2.json | 1220 +++++++++++++++++ .../datasource/cloudwatch/datasource.ts | 339 +++-- .../datasource/cloudwatch/memoizedDebounce.ts | 13 + .../plugins/datasource/cloudwatch/module.ts | 16 - .../plugins/datasource/cloudwatch/module.tsx | 16 + .../cloudwatch/partials/config.html | 55 - .../cloudwatch/partials/query.editor.html | 4 - .../cloudwatch/partials/query.parameter.html | 193 ++- .../plugins/datasource/cloudwatch/plugin.json | 1 - .../datasource/cloudwatch/query_ctrl.ts | 15 - .../cloudwatch/specs/datasource.test.ts | 298 ++-- .../plugins/datasource/cloudwatch/types.ts | 21 +- public/app/types/appNotifications.ts | 1 + scripts/go/go.mod | 5 +- 62 files changed, 7527 insertions(+), 1512 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/cloudwatch_query.go create mode 100644 pkg/tsdb/cloudwatch/cloudwatch_query_test.go delete mode 100644 pkg/tsdb/cloudwatch/cloudwatch_test.go delete mode 100644 pkg/tsdb/cloudwatch/constants.go delete mode 100644 pkg/tsdb/cloudwatch/get_metric_data.go create mode 100644 pkg/tsdb/cloudwatch/get_metric_data_executor.go create mode 100644 pkg/tsdb/cloudwatch/get_metric_data_executor_test.go delete mode 100644 pkg/tsdb/cloudwatch/get_metric_data_test.go delete mode 100644 pkg/tsdb/cloudwatch/get_metric_statistics.go delete mode 100644 pkg/tsdb/cloudwatch/get_metric_statistics_test.go create mode 100644 pkg/tsdb/cloudwatch/metric_data_input_builder.go create mode 100644 pkg/tsdb/cloudwatch/metric_data_input_builder_test.go create mode 100644 pkg/tsdb/cloudwatch/metric_data_query_builder.go create mode 100644 pkg/tsdb/cloudwatch/metric_data_query_builder_test.go create mode 100644 pkg/tsdb/cloudwatch/query_transformer.go create mode 100644 pkg/tsdb/cloudwatch/query_transformer_test.go create mode 100644 pkg/tsdb/cloudwatch/request_parser.go create mode 100644 pkg/tsdb/cloudwatch/request_parser_test.go create mode 100644 pkg/tsdb/cloudwatch/response_parser.go create mode 100644 pkg/tsdb/cloudwatch/response_parser_test.go create mode 100644 pkg/tsdb/cloudwatch/time_series_query.go create mode 100644 public/app/plugins/datasource/cloudwatch/components/Alias.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Alias.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/ConfigEditor.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/ConfigEditor.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Dimensions.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Dimensions.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Forms.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/QueryEditor.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/QueryEditor.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Stats.test.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/Stats.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/ThrottlingErrorMessage.tsx create mode 100644 public/app/plugins/datasource/cloudwatch/components/__snapshots__/Alias.test.tsx.snap create mode 100644 public/app/plugins/datasource/cloudwatch/components/__snapshots__/ConfigEditor.test.tsx.snap create mode 100644 public/app/plugins/datasource/cloudwatch/components/__snapshots__/QueryEditor.test.tsx.snap create mode 100644 public/app/plugins/datasource/cloudwatch/components/__snapshots__/Stats.test.tsx.snap create mode 100644 public/app/plugins/datasource/cloudwatch/components/index.ts delete mode 100644 public/app/plugins/datasource/cloudwatch/config_ctrl.ts create mode 100644 public/app/plugins/datasource/cloudwatch/dashboards/EBS.json create mode 100644 public/app/plugins/datasource/cloudwatch/dashboards/Lambda.json create mode 100644 public/app/plugins/datasource/cloudwatch/dashboards/ec2.json create mode 100644 public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts delete mode 100644 public/app/plugins/datasource/cloudwatch/module.ts create mode 100644 public/app/plugins/datasource/cloudwatch/module.tsx delete mode 100644 public/app/plugins/datasource/cloudwatch/partials/config.html delete mode 100644 public/app/plugins/datasource/cloudwatch/partials/query.editor.html delete mode 100644 public/app/plugins/datasource/cloudwatch/query_ctrl.ts diff --git a/pkg/tsdb/cloudwatch/annotation_query.go b/pkg/tsdb/cloudwatch/annotation_query.go index e0d9158435e..7c6465f278c 100644 --- a/pkg/tsdb/cloudwatch/annotation_query.go +++ b/pkg/tsdb/cloudwatch/annotation_query.go @@ -24,7 +24,7 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo namespace := parameters.Get("namespace").MustString("") metricName := parameters.Get("metricName").MustString("") dimensions := parameters.Get("dimensions").MustMap() - statistics, extendedStatistics, err := parseStatistics(parameters) + statistics, err := parseStatistics(parameters) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo if err != nil { return nil, errors.New("Failed to call cloudwatch:DescribeAlarms") } - alarmNames = filterAlarms(resp, namespace, metricName, dimensions, statistics, extendedStatistics, period) + alarmNames = filterAlarms(resp, namespace, metricName, dimensions, statistics, period) } else { if region == "" || namespace == "" || metricName == "" || len(statistics) == 0 { return result, nil @@ -82,22 +82,6 @@ func (e *CloudWatchExecutor) executeAnnotationQuery(ctx context.Context, queryCo alarmNames = append(alarmNames, alarm.AlarmName) } } - for _, s := range extendedStatistics { - params := &cloudwatch.DescribeAlarmsForMetricInput{ - Namespace: aws.String(namespace), - MetricName: aws.String(metricName), - Dimensions: qd, - ExtendedStatistic: aws.String(s), - Period: aws.Int64(period), - } - resp, err := svc.DescribeAlarmsForMetric(params) - if err != nil { - return nil, errors.New("Failed to call cloudwatch:DescribeAlarmsForMetric") - } - for _, alarm := range resp.MetricAlarms { - alarmNames = append(alarmNames, alarm.AlarmName) - } - } } startTime, err := queryContext.TimeRange.ParseFrom() @@ -158,7 +142,7 @@ func transformAnnotationToTable(data []map[string]string, result *tsdb.QueryResu result.Meta.Set("rowCount", len(data)) } -func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, metricName string, dimensions map[string]interface{}, statistics []string, extendedStatistics []string, period int64) []*string { +func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, metricName string, dimensions map[string]interface{}, statistics []string, period int64) []*string { alarmNames := make([]*string, 0) for _, alarm := range alarms.MetricAlarms { @@ -197,18 +181,6 @@ func filterAlarms(alarms *cloudwatch.DescribeAlarmsOutput, namespace string, met } } - if len(extendedStatistics) != 0 { - found := false - for _, s := range extendedStatistics { - if *alarm.Statistic == s { - found = true - } - } - if !found { - continue - } - } - if period != 0 && *alarm.Period != period { continue } diff --git a/pkg/tsdb/cloudwatch/cloudwatch.go b/pkg/tsdb/cloudwatch/cloudwatch.go index 34d57f4d71f..54b36a73357 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch.go +++ b/pkg/tsdb/cloudwatch/cloudwatch.go @@ -2,18 +2,13 @@ package cloudwatch import ( "context" - "fmt" "regexp" - "strconv" - "strings" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" - "golang.org/x/sync/errgroup" ) type CloudWatchExecutor struct { @@ -38,21 +33,13 @@ func NewCloudWatchExecutor(dsInfo *models.DataSource) (tsdb.TsdbQueryEndpoint, e } var ( - plog log.Logger - standardStatistics map[string]bool - aliasFormat *regexp.Regexp + plog log.Logger + aliasFormat *regexp.Regexp ) func init() { plog = log.New("tsdb.cloudwatch") tsdb.RegisterTsdbQueryEndpoint("cloudwatch", NewCloudWatchExecutor) - standardStatistics = map[string]bool{ - "Average": true, - "Maximum": true, - "Minimum": true, - "Sum": true, - "SampleCount": true, - } aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`) } @@ -75,162 +62,3 @@ func (e *CloudWatchExecutor) Query(ctx context.Context, dsInfo *models.DataSourc return result, err } - -func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { - results := &tsdb.Response{ - Results: make(map[string]*tsdb.QueryResult), - } - resultChan := make(chan *tsdb.QueryResult, len(queryContext.Queries)) - - eg, ectx := errgroup.WithContext(ctx) - - getMetricDataQueries := make(map[string]map[string]*CloudWatchQuery) - for i, model := range queryContext.Queries { - queryType := model.Model.Get("type").MustString() - if queryType != "timeSeriesQuery" && queryType != "" { - continue - } - - RefId := queryContext.Queries[i].RefId - query, err := parseQuery(queryContext.Queries[i].Model) - if err != nil { - results.Results[RefId] = &tsdb.QueryResult{ - Error: err, - } - return results, nil - } - query.RefId = RefId - - if query.Id != "" { - if _, ok := getMetricDataQueries[query.Region]; !ok { - getMetricDataQueries[query.Region] = make(map[string]*CloudWatchQuery) - } - getMetricDataQueries[query.Region][query.Id] = query - continue - } - - if query.Id == "" && query.Expression != "" { - results.Results[query.RefId] = &tsdb.QueryResult{ - Error: fmt.Errorf("Invalid query: id should be set if using expression"), - } - return results, nil - } - - eg.Go(func() error { - defer func() { - if err := recover(); err != nil { - plog.Error("Execute Query Panic", "error", err, "stack", log.Stack(1)) - if theErr, ok := err.(error); ok { - resultChan <- &tsdb.QueryResult{ - RefId: query.RefId, - Error: theErr, - } - } - } - }() - - queryRes, err := e.executeQuery(ectx, query, queryContext) - if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" { - return err - } - if err != nil { - resultChan <- &tsdb.QueryResult{ - RefId: query.RefId, - Error: err, - } - return nil - } - resultChan <- queryRes - return nil - }) - } - - if len(getMetricDataQueries) > 0 { - for region, getMetricDataQuery := range getMetricDataQueries { - q := getMetricDataQuery - eg.Go(func() error { - defer func() { - if err := recover(); err != nil { - plog.Error("Execute Get Metric Data Query Panic", "error", err, "stack", log.Stack(1)) - if theErr, ok := err.(error); ok { - resultChan <- &tsdb.QueryResult{ - Error: theErr, - } - } - } - }() - - queryResponses, err := e.executeGetMetricDataQuery(ectx, region, q, queryContext) - if ae, ok := err.(awserr.Error); ok && ae.Code() == "500" { - return err - } - for _, queryRes := range queryResponses { - if err != nil { - queryRes.Error = err - } - resultChan <- queryRes - } - return nil - }) - } - } - - if err := eg.Wait(); err != nil { - return nil, err - } - close(resultChan) - for result := range resultChan { - results.Results[result.RefId] = result - } - - return results, nil -} - -func formatAlias(query *CloudWatchQuery, stat string, dimensions map[string]string, label string) string { - region := query.Region - namespace := query.Namespace - metricName := query.MetricName - period := strconv.Itoa(query.Period) - if len(query.Id) > 0 && len(query.Expression) > 0 { - if strings.Index(query.Expression, "SEARCH(") == 0 { - pIndex := strings.LastIndex(query.Expression, ",") - period = strings.Trim(query.Expression[pIndex+1:], " )") - sIndex := strings.LastIndex(query.Expression[:pIndex], ",") - stat = strings.Trim(query.Expression[sIndex+1:pIndex], " '") - } else if len(query.Alias) > 0 { - // expand by Alias - } else { - return query.Id - } - } - - data := map[string]string{} - data["region"] = region - data["namespace"] = namespace - data["metric"] = metricName - data["stat"] = stat - data["period"] = period - if len(label) != 0 { - data["label"] = label - } - for k, v := range dimensions { - data[k] = v - } - - result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte { - labelName := strings.Replace(string(in), "{{", "", 1) - labelName = strings.Replace(labelName, "}}", "", 1) - labelName = strings.TrimSpace(labelName) - if val, exists := data[labelName]; exists { - return []byte(val) - } - - return in - }) - - if string(result) == "" { - return metricName + "_" + stat - } - - return string(result) -} diff --git a/pkg/tsdb/cloudwatch/cloudwatch_query.go b/pkg/tsdb/cloudwatch/cloudwatch_query.go new file mode 100644 index 00000000000..fdb62373fbe --- /dev/null +++ b/pkg/tsdb/cloudwatch/cloudwatch_query.go @@ -0,0 +1,62 @@ +package cloudwatch + +import ( + "strings" +) + +type cloudWatchQuery struct { + RefId string + Region string + Id string + Namespace string + MetricName string + Stats string + Expression string + ReturnData bool + Dimensions map[string][]string + Period int + Alias string + Identifier string + HighResolution bool + MatchExact bool + UsedExpression string + RequestExceededMaxLimit bool +} + +func (q *cloudWatchQuery) isMathExpression() bool { + return q.Expression != "" && !q.isUserDefinedSearchExpression() +} + +func (q *cloudWatchQuery) isSearchExpression() bool { + return q.isUserDefinedSearchExpression() || q.isInferredSearchExpression() +} + +func (q *cloudWatchQuery) isUserDefinedSearchExpression() bool { + return strings.Contains(q.Expression, "SEARCH(") +} + +func (q *cloudWatchQuery) isInferredSearchExpression() bool { + if len(q.Dimensions) == 0 { + return !q.MatchExact + } + + if !q.MatchExact { + return true + } + + for _, values := range q.Dimensions { + if len(values) > 1 { + return true + } + for _, v := range values { + if v == "*" { + return true + } + } + } + return false +} + +func (q *cloudWatchQuery) isMetricStat() bool { + return !q.isSearchExpression() && !q.isMathExpression() +} diff --git a/pkg/tsdb/cloudwatch/cloudwatch_query_test.go b/pkg/tsdb/cloudwatch/cloudwatch_query_test.go new file mode 100644 index 00000000000..f061d1c47c9 --- /dev/null +++ b/pkg/tsdb/cloudwatch/cloudwatch_query_test.go @@ -0,0 +1,175 @@ +package cloudwatch + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestCloudWatchQuery(t *testing.T) { + Convey("TestCloudWatchQuery", t, func() { + Convey("and SEARCH(someexpression) was specified in the query editor", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "SEARCH(someexpression)", + Stats: "Average", + Period: 300, + Id: "id1", + Identifier: "id1", + } + + Convey("it is a search expression", func() { + So(query.isSearchExpression(), ShouldBeTrue) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + }) + + Convey("and no expression, no multi dimension key values and no * was used", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Stats: "Average", + Period: 300, + Id: "id1", + Identifier: "id1", + MatchExact: true, + Dimensions: map[string][]string{ + "InstanceId": {"i-12345678"}, + }, + } + + Convey("it is not a search expression", func() { + So(query.isSearchExpression(), ShouldBeFalse) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + }) + + Convey("and no expression but multi dimension key values exist", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Stats: "Average", + Period: 300, + Id: "id1", + Identifier: "id1", + Dimensions: map[string][]string{ + "InstanceId": {"i-12345678", "i-34562312"}, + }, + } + + Convey("it is a search expression", func() { + So(query.isSearchExpression(), ShouldBeTrue) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + }) + + Convey("and no expression but dimension values has *", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Stats: "Average", + Period: 300, + Id: "id1", + Identifier: "id1", + Dimensions: map[string][]string{ + "InstanceId": {"i-12345678", "*"}, + "InstanceType": {"abc", "def"}, + }, + } + + Convey("it is not a search expression", func() { + So(query.isSearchExpression(), ShouldBeTrue) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + }) + + Convey("and no dimensions were added", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Stats: "Average", + Period: 300, + Id: "id1", + MatchExact: false, + Identifier: "id1", + Dimensions: make(map[string][]string), + } + Convey("and match exact is false", func() { + query.MatchExact = false + Convey("it is a search expression", func() { + So(query.isSearchExpression(), ShouldBeTrue) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + + Convey("it is not metric stat", func() { + So(query.isMetricStat(), ShouldBeFalse) + }) + + }) + + Convey("and match exact is true", func() { + query.MatchExact = true + Convey("it is a search expression", func() { + So(query.isSearchExpression(), ShouldBeFalse) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + + Convey("it is a metric stat", func() { + So(query.isMetricStat(), ShouldBeTrue) + }) + + }) + }) + + Convey("and match exact is", func() { + query := &cloudWatchQuery{ + RefId: "A", + Region: "us-east-1", + Expression: "", + Stats: "Average", + Period: 300, + Id: "id1", + Identifier: "id1", + MatchExact: false, + Dimensions: map[string][]string{ + "InstanceId": {"i-12345678"}, + }, + } + + Convey("it is a search expression", func() { + So(query.isSearchExpression(), ShouldBeTrue) + }) + + Convey("it is not math expressions", func() { + So(query.isMathExpression(), ShouldBeFalse) + }) + + Convey("it is not metric stat", func() { + So(query.isMetricStat(), ShouldBeFalse) + }) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go deleted file mode 100644 index 9d8a4a88006..00000000000 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package cloudwatch - -import ( - "context" - "testing" - - "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/tsdb" - - "github.com/grafana/grafana/pkg/components/simplejson" - . "github.com/smartystreets/goconvey/convey" -) - -func TestCloudWatch(t *testing.T) { - Convey("CloudWatch", t, func() { - - Convey("executeQuery", func() { - e := &CloudWatchExecutor{ - DataSource: &models.DataSource{ - JsonData: simplejson.New(), - }, - } - - Convey("End time before start time should result in error", func() { - _, err := e.executeQuery(context.Background(), &CloudWatchQuery{}, &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")}) - So(err.Error(), ShouldEqual, "Invalid time range: Start time must be before end time") - }) - - Convey("End time equals start time should result in error", func() { - _, err := e.executeQuery(context.Background(), &CloudWatchQuery{}, &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-1h")}) - So(err.Error(), ShouldEqual, "Invalid time range: Start time must be before end time") - }) - }) - }) -} diff --git a/pkg/tsdb/cloudwatch/constants.go b/pkg/tsdb/cloudwatch/constants.go deleted file mode 100644 index 23817b1d133..00000000000 --- a/pkg/tsdb/cloudwatch/constants.go +++ /dev/null @@ -1,30 +0,0 @@ -package cloudwatch - -var cloudwatchUnitMappings = map[string]string{ - "Seconds": "s", - "Microseconds": "µs", - "Milliseconds": "ms", - "Bytes": "bytes", - "Kilobytes": "kbytes", - "Megabytes": "mbytes", - "Gigabytes": "gbytes", - //"Terabytes": "", - "Bits": "bits", - //"Kilobits": "", - //"Megabits": "", - //"Gigabits": "", - //"Terabits": "", - "Percent": "percent", - //"Count": "", - "Bytes/Second": "Bps", - "Kilobytes/Second": "KBs", - "Megabytes/Second": "MBs", - "Gigabytes/Second": "GBs", - //"Terabytes/Second": "", - "Bits/Second": "bps", - "Kilobits/Second": "Kbits", - "Megabits/Second": "Mbits", - "Gigabits/Second": "Gbits", - //"Terabits/Second": "", - //"Count/Second": "", -} diff --git a/pkg/tsdb/cloudwatch/credentials.go b/pkg/tsdb/cloudwatch/credentials.go index fb92c827f7c..414d1995b7b 100644 --- a/pkg/tsdb/cloudwatch/credentials.go +++ b/pkg/tsdb/cloudwatch/credentials.go @@ -12,9 +12,11 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatch" "github.com/aws/aws-sdk-go/service/sts" + "github.com/grafana/grafana/pkg/setting" ) type cache struct { @@ -180,6 +182,7 @@ func (e *CloudWatchExecutor) getAwsConfig(dsInfo *DatasourceInfo) (*aws.Config, Region: aws.String(dsInfo.Region), Credentials: creds, } + return cfg, nil } @@ -196,5 +199,10 @@ func (e *CloudWatchExecutor) getClient(region string) (*cloudwatch.CloudWatch, e } client := cloudwatch.New(sess, cfg) + + client.Handlers.Send.PushFront(func(r *request.Request) { + r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion)) + }) + return client, nil } diff --git a/pkg/tsdb/cloudwatch/get_metric_data.go b/pkg/tsdb/cloudwatch/get_metric_data.go deleted file mode 100644 index 22f36d18596..00000000000 --- a/pkg/tsdb/cloudwatch/get_metric_data.go +++ /dev/null @@ -1,171 +0,0 @@ -package cloudwatch - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/components/null" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/metrics" - "github.com/grafana/grafana/pkg/tsdb" -) - -func (e *CloudWatchExecutor) executeGetMetricDataQuery(ctx context.Context, region string, queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) ([]*tsdb.QueryResult, error) { - queryResponses := make([]*tsdb.QueryResult, 0) - - client, err := e.getClient(region) - if err != nil { - return queryResponses, err - } - - params, err := parseGetMetricDataQuery(queries, queryContext) - if err != nil { - return queryResponses, err - } - - nextToken := "" - mdr := make(map[string]map[string]*cloudwatch.MetricDataResult) - for { - if nextToken != "" { - params.NextToken = aws.String(nextToken) - } - resp, err := client.GetMetricDataWithContext(ctx, params) - if err != nil { - return queryResponses, err - } - metrics.MAwsCloudWatchGetMetricData.Add(float64(len(params.MetricDataQueries))) - - for _, r := range resp.MetricDataResults { - if _, ok := mdr[*r.Id]; !ok { - mdr[*r.Id] = make(map[string]*cloudwatch.MetricDataResult) - mdr[*r.Id][*r.Label] = r - } else if _, ok := mdr[*r.Id][*r.Label]; !ok { - mdr[*r.Id][*r.Label] = r - } else { - mdr[*r.Id][*r.Label].Timestamps = append(mdr[*r.Id][*r.Label].Timestamps, r.Timestamps...) - mdr[*r.Id][*r.Label].Values = append(mdr[*r.Id][*r.Label].Values, r.Values...) - } - } - - if resp.NextToken == nil || *resp.NextToken == "" { - break - } - nextToken = *resp.NextToken - } - - for id, lr := range mdr { - queryRes, err := parseGetMetricDataResponse(lr, queries[id]) - if err != nil { - return queryResponses, err - } - queryResponses = append(queryResponses, queryRes) - } - - return queryResponses, nil -} - -func parseGetMetricDataQuery(queries map[string]*CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*cloudwatch.GetMetricDataInput, error) { - // validate query - for _, query := range queries { - if !(len(query.Statistics) == 1 && len(query.ExtendedStatistics) == 0) && - !(len(query.Statistics) == 0 && len(query.ExtendedStatistics) == 1) { - return nil, errors.New("Statistics count should be 1") - } - } - - startTime, err := queryContext.TimeRange.ParseFrom() - if err != nil { - return nil, err - } - - endTime, err := queryContext.TimeRange.ParseTo() - if err != nil { - return nil, err - } - - params := &cloudwatch.GetMetricDataInput{ - StartTime: aws.Time(startTime), - EndTime: aws.Time(endTime), - ScanBy: aws.String("TimestampAscending"), - } - for _, query := range queries { - // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 - if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { - return nil, errors.New("too long query period") - } - - mdq := &cloudwatch.MetricDataQuery{ - Id: aws.String(query.Id), - ReturnData: aws.Bool(query.ReturnData), - } - if query.Expression != "" { - mdq.Expression = aws.String(query.Expression) - } else { - mdq.MetricStat = &cloudwatch.MetricStat{ - Metric: &cloudwatch.Metric{ - Namespace: aws.String(query.Namespace), - MetricName: aws.String(query.MetricName), - }, - Period: aws.Int64(int64(query.Period)), - } - for _, d := range query.Dimensions { - mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions, - &cloudwatch.Dimension{ - Name: d.Name, - Value: d.Value, - }) - } - if len(query.Statistics) == 1 { - mdq.MetricStat.Stat = query.Statistics[0] - } else { - mdq.MetricStat.Stat = query.ExtendedStatistics[0] - } - } - params.MetricDataQueries = append(params.MetricDataQueries, mdq) - } - return params, nil -} - -func parseGetMetricDataResponse(lr map[string]*cloudwatch.MetricDataResult, query *CloudWatchQuery) (*tsdb.QueryResult, error) { - queryRes := tsdb.NewQueryResult() - queryRes.RefId = query.RefId - - for label, r := range lr { - if *r.StatusCode != "Complete" { - return queryRes, fmt.Errorf("Part of query is failed: %s", *r.StatusCode) - } - - series := tsdb.TimeSeries{ - Tags: map[string]string{}, - Points: make([]tsdb.TimePoint, 0), - } - for _, d := range query.Dimensions { - series.Tags[*d.Name] = *d.Value - } - s := "" - if len(query.Statistics) == 1 { - s = *query.Statistics[0] - } else { - s = *query.ExtendedStatistics[0] - } - series.Name = formatAlias(query, s, series.Tags, label) - - for j, t := range r.Timestamps { - if j > 0 { - expectedTimestamp := r.Timestamps[j-1].Add(time.Duration(query.Period) * time.Second) - if expectedTimestamp.Before(*t) { - series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000))) - } - } - series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*r.Values[j]), float64((*t).Unix())*1000)) - } - - queryRes.Series = append(queryRes.Series, &series) - queryRes.Meta = simplejson.New() - } - return queryRes, nil -} diff --git a/pkg/tsdb/cloudwatch/get_metric_data_executor.go b/pkg/tsdb/cloudwatch/get_metric_data_executor.go new file mode 100644 index 00000000000..d8df196c560 --- /dev/null +++ b/pkg/tsdb/cloudwatch/get_metric_data_executor.go @@ -0,0 +1,34 @@ +package cloudwatch + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/infra/metrics" +) + +func (e *CloudWatchExecutor) executeRequest(ctx context.Context, client cloudWatchClient, metricDataInput *cloudwatch.GetMetricDataInput) ([]*cloudwatch.GetMetricDataOutput, error) { + mdo := make([]*cloudwatch.GetMetricDataOutput, 0) + + nextToken := "" + for { + if nextToken != "" { + metricDataInput.NextToken = aws.String(nextToken) + } + resp, err := client.GetMetricDataWithContext(ctx, metricDataInput) + if err != nil { + return mdo, err + } + + mdo = append(mdo, resp) + metrics.MAwsCloudWatchGetMetricData.Add(float64(len(metricDataInput.MetricDataQueries))) + + if resp.NextToken == nil || *resp.NextToken == "" { + break + } + nextToken = *resp.NextToken + } + + return mdo, nil +} diff --git a/pkg/tsdb/cloudwatch/get_metric_data_executor_test.go b/pkg/tsdb/cloudwatch/get_metric_data_executor_test.go new file mode 100644 index 00000000000..6c3991b11aa --- /dev/null +++ b/pkg/tsdb/cloudwatch/get_metric_data_executor_test.go @@ -0,0 +1,50 @@ +package cloudwatch + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/cloudwatch" + . "github.com/smartystreets/goconvey/convey" +) + +var counter = 1 + +type cloudWatchFakeClient struct { +} + +func (client *cloudWatchFakeClient) GetMetricDataWithContext(ctx aws.Context, input *cloudwatch.GetMetricDataInput, opts ...request.Option) (*cloudwatch.GetMetricDataOutput, error) { + nextToken := "next" + res := []*cloudwatch.MetricDataResult{{ + Values: []*float64{aws.Float64(12.3), aws.Float64(23.5)}, + }} + if counter == 0 { + nextToken = "" + res = []*cloudwatch.MetricDataResult{{ + Values: []*float64{aws.Float64(100)}, + }} + } + counter-- + return &cloudwatch.GetMetricDataOutput{ + MetricDataResults: res, + NextToken: aws.String(nextToken), + }, nil +} + +func TestGetMetricDataExecutorTest(t *testing.T) { + Convey("TestGetMetricDataExecutorTest", t, func() { + Convey("pagination works", func() { + executor := &CloudWatchExecutor{} + inputs := &cloudwatch.GetMetricDataInput{MetricDataQueries: []*cloudwatch.MetricDataQuery{}} + res, err := executor.executeRequest(context.Background(), &cloudWatchFakeClient{}, inputs) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 2) + So(len(res[0].MetricDataResults[0].Values), ShouldEqual, 2) + So(*res[0].MetricDataResults[0].Values[1], ShouldEqual, 23.5) + So(*res[1].MetricDataResults[0].Values[0], ShouldEqual, 100) + }) + }) + +} diff --git a/pkg/tsdb/cloudwatch/get_metric_data_test.go b/pkg/tsdb/cloudwatch/get_metric_data_test.go deleted file mode 100644 index ae8b4090e4a..00000000000 --- a/pkg/tsdb/cloudwatch/get_metric_data_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package cloudwatch - -import ( - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/components/null" - "github.com/grafana/grafana/pkg/tsdb" - . "github.com/smartystreets/goconvey/convey" -) - -func TestCloudWatchGetMetricData(t *testing.T) { - Convey("CloudWatchGetMetricData", t, func() { - - Convey("can parse cloudwatch GetMetricData query", func() { - queries := map[string]*CloudWatchQuery{ - "id1": { - RefId: "A", - Region: "us-east-1", - Namespace: "AWS/EC2", - MetricName: "CPUUtilization", - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("InstanceId"), - Value: aws.String("i-12345678"), - }, - }, - Statistics: []*string{aws.String("Average")}, - Period: 300, - Id: "id1", - Expression: "", - }, - "id2": { - RefId: "B", - Region: "us-east-1", - Statistics: []*string{aws.String("Average")}, - Id: "id2", - Expression: "id1 * 2", - }, - } - - queryContext := &tsdb.TsdbQuery{ - TimeRange: tsdb.NewFakeTimeRange("5m", "now", time.Now()), - } - res, err := parseGetMetricDataQuery(queries, queryContext) - So(err, ShouldBeNil) - - for _, v := range res.MetricDataQueries { - if *v.Id == "id1" { - So(*v.MetricStat.Metric.Namespace, ShouldEqual, "AWS/EC2") - So(*v.MetricStat.Metric.MetricName, ShouldEqual, "CPUUtilization") - So(*v.MetricStat.Metric.Dimensions[0].Name, ShouldEqual, "InstanceId") - So(*v.MetricStat.Metric.Dimensions[0].Value, ShouldEqual, "i-12345678") - So(*v.MetricStat.Period, ShouldEqual, 300) - So(*v.MetricStat.Stat, ShouldEqual, "Average") - So(*v.Id, ShouldEqual, "id1") - } else { - So(*v.Id, ShouldEqual, "id2") - So(*v.Expression, ShouldEqual, "id1 * 2") - } - } - }) - - Convey("can parse cloudwatch response", func() { - timestamp := time.Unix(0, 0) - resp := map[string]*cloudwatch.MetricDataResult{ - "label": { - Id: aws.String("id1"), - Label: aws.String("label"), - Timestamps: []*time.Time{ - aws.Time(timestamp), - aws.Time(timestamp.Add(60 * time.Second)), - aws.Time(timestamp.Add(180 * time.Second)), - }, - Values: []*float64{ - aws.Float64(10), - aws.Float64(20), - aws.Float64(30), - }, - StatusCode: aws.String("Complete"), - }, - } - query := &CloudWatchQuery{ - RefId: "refId1", - Region: "us-east-1", - Namespace: "AWS/ApplicationELB", - MetricName: "TargetResponseTime", - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("LoadBalancer"), - Value: aws.String("lb"), - }, - { - Name: aws.String("TargetGroup"), - Value: aws.String("tg"), - }, - }, - Statistics: []*string{aws.String("Average")}, - Period: 60, - Alias: "{{namespace}}_{{metric}}_{{stat}}", - } - queryRes, err := parseGetMetricDataResponse(resp, query) - So(err, ShouldBeNil) - So(queryRes.RefId, ShouldEqual, "refId1") - So(queryRes.Series[0].Name, ShouldEqual, "AWS/ApplicationELB_TargetResponseTime_Average") - So(queryRes.Series[0].Tags["LoadBalancer"], ShouldEqual, "lb") - So(queryRes.Series[0].Tags["TargetGroup"], ShouldEqual, "tg") - So(queryRes.Series[0].Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) - So(queryRes.Series[0].Points[1][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) - So(queryRes.Series[0].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) - So(queryRes.Series[0].Points[3][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) - }) - }) -} diff --git a/pkg/tsdb/cloudwatch/get_metric_statistics.go b/pkg/tsdb/cloudwatch/get_metric_statistics.go deleted file mode 100644 index ded80670e13..00000000000 --- a/pkg/tsdb/cloudwatch/get_metric_statistics.go +++ /dev/null @@ -1,276 +0,0 @@ -package cloudwatch - -import ( - "context" - "errors" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/components/null" - "github.com/grafana/grafana/pkg/components/simplejson" - "github.com/grafana/grafana/pkg/infra/metrics" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb" -) - -func (e *CloudWatchExecutor) executeQuery(ctx context.Context, query *CloudWatchQuery, queryContext *tsdb.TsdbQuery) (*tsdb.QueryResult, error) { - client, err := e.getClient(query.Region) - if err != nil { - return nil, err - } - - startTime, err := queryContext.TimeRange.ParseFrom() - if err != nil { - return nil, err - } - - endTime, err := queryContext.TimeRange.ParseTo() - if err != nil { - return nil, err - } - - if !startTime.Before(endTime) { - return nil, fmt.Errorf("Invalid time range: Start time must be before end time") - } - - params := &cloudwatch.GetMetricStatisticsInput{ - Namespace: aws.String(query.Namespace), - MetricName: aws.String(query.MetricName), - Dimensions: query.Dimensions, - Period: aws.Int64(int64(query.Period)), - } - if len(query.Statistics) > 0 { - params.Statistics = query.Statistics - } - if len(query.ExtendedStatistics) > 0 { - params.ExtendedStatistics = query.ExtendedStatistics - } - - // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 - if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { - return nil, errors.New("too long query period") - } - var resp *cloudwatch.GetMetricStatisticsOutput - for startTime.Before(endTime) { - params.StartTime = aws.Time(startTime) - if query.HighResolution { - startTime = startTime.Add(time.Duration(1440*query.Period) * time.Second) - } else { - startTime = endTime - } - params.EndTime = aws.Time(startTime) - - if setting.Env == setting.DEV { - plog.Debug("CloudWatch query", "raw query", params) - } - - partResp, err := client.GetMetricStatisticsWithContext(ctx, params, request.WithResponseReadTimeout(10*time.Second)) - if err != nil { - return nil, err - } - if resp != nil { - resp.Datapoints = append(resp.Datapoints, partResp.Datapoints...) - } else { - resp = partResp - - } - metrics.MAwsCloudWatchGetMetricStatistics.Inc() - } - - queryRes, err := parseResponse(resp, query) - if err != nil { - return nil, err - } - - return queryRes, nil -} - -func parseQuery(model *simplejson.Json) (*CloudWatchQuery, error) { - region, err := model.Get("region").String() - if err != nil { - return nil, err - } - - namespace, err := model.Get("namespace").String() - if err != nil { - return nil, err - } - - metricName, err := model.Get("metricName").String() - if err != nil { - return nil, err - } - - id := model.Get("id").MustString("") - expression := model.Get("expression").MustString("") - - dimensions, err := parseDimensions(model) - if err != nil { - return nil, err - } - - statistics, extendedStatistics, err := parseStatistics(model) - if err != nil { - return nil, err - } - - p := model.Get("period").MustString("") - if p == "" { - if namespace == "AWS/EC2" { - p = "300" - } else { - p = "60" - } - } - - var period int - if regexp.MustCompile(`^\d+$`).Match([]byte(p)) { - period, err = strconv.Atoi(p) - if err != nil { - return nil, err - } - } else { - d, err := time.ParseDuration(p) - if err != nil { - return nil, err - } - period = int(d.Seconds()) - } - - alias := model.Get("alias").MustString() - - returnData := !model.Get("hide").MustBool(false) - queryType := model.Get("type").MustString() - if queryType == "" { - // If no type is provided we assume we are called by alerting service, which requires to return data! - // Note, this is sort of a hack, but the official Grafana interfaces do not carry the information - // who (which service) called the TsdbQueryEndpoint.Query(...) function. - returnData = true - } - highResolution := model.Get("highResolution").MustBool(false) - - return &CloudWatchQuery{ - Region: region, - Namespace: namespace, - MetricName: metricName, - Dimensions: dimensions, - Statistics: aws.StringSlice(statistics), - ExtendedStatistics: aws.StringSlice(extendedStatistics), - Period: period, - Alias: alias, - Id: id, - Expression: expression, - ReturnData: returnData, - HighResolution: highResolution, - }, nil -} - -func parseDimensions(model *simplejson.Json) ([]*cloudwatch.Dimension, error) { - var result []*cloudwatch.Dimension - - for k, v := range model.Get("dimensions").MustMap() { - kk := k - if vv, ok := v.(string); ok { - result = append(result, &cloudwatch.Dimension{ - Name: &kk, - Value: &vv, - }) - } else { - return nil, errors.New("failed to parse") - } - } - - sort.Slice(result, func(i, j int) bool { - return *result[i].Name < *result[j].Name - }) - return result, nil -} - -func parseStatistics(model *simplejson.Json) ([]string, []string, error) { - var statistics []string - var extendedStatistics []string - - for _, s := range model.Get("statistics").MustArray() { - if ss, ok := s.(string); ok { - if _, isStandard := standardStatistics[ss]; isStandard { - statistics = append(statistics, ss) - } else { - extendedStatistics = append(extendedStatistics, ss) - } - } else { - return nil, nil, errors.New("failed to parse") - } - } - - return statistics, extendedStatistics, nil -} - -func parseResponse(resp *cloudwatch.GetMetricStatisticsOutput, query *CloudWatchQuery) (*tsdb.QueryResult, error) { - queryRes := tsdb.NewQueryResult() - - queryRes.RefId = query.RefId - var value float64 - for _, s := range append(query.Statistics, query.ExtendedStatistics...) { - series := tsdb.TimeSeries{ - Tags: map[string]string{}, - Points: make([]tsdb.TimePoint, 0), - } - for _, d := range query.Dimensions { - series.Tags[*d.Name] = *d.Value - } - series.Name = formatAlias(query, *s, series.Tags, "") - - lastTimestamp := make(map[string]time.Time) - sort.Slice(resp.Datapoints, func(i, j int) bool { - return (*resp.Datapoints[i].Timestamp).Before(*resp.Datapoints[j].Timestamp) - }) - for _, v := range resp.Datapoints { - switch *s { - case "Average": - value = *v.Average - case "Maximum": - value = *v.Maximum - case "Minimum": - value = *v.Minimum - case "Sum": - value = *v.Sum - case "SampleCount": - value = *v.SampleCount - default: - if strings.Index(*s, "p") == 0 && v.ExtendedStatistics[*s] != nil { - value = *v.ExtendedStatistics[*s] - } - } - - // terminate gap of data points - timestamp := *v.Timestamp - if _, ok := lastTimestamp[*s]; ok { - nextTimestampFromLast := lastTimestamp[*s].Add(time.Duration(query.Period) * time.Second) - for timestamp.After(nextTimestampFromLast) { - series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(nextTimestampFromLast.Unix()*1000))) - nextTimestampFromLast = nextTimestampFromLast.Add(time.Duration(query.Period) * time.Second) - } - } - lastTimestamp[*s] = timestamp - - series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(value), float64(timestamp.Unix()*1000))) - } - - queryRes.Series = append(queryRes.Series, &series) - queryRes.Meta = simplejson.New() - if len(resp.Datapoints) > 0 && resp.Datapoints[0].Unit != nil { - if unit, ok := cloudwatchUnitMappings[*resp.Datapoints[0].Unit]; ok { - queryRes.Meta.Set("unit", unit) - } - } - } - - return queryRes, nil -} diff --git a/pkg/tsdb/cloudwatch/get_metric_statistics_test.go b/pkg/tsdb/cloudwatch/get_metric_statistics_test.go deleted file mode 100644 index 373cdb097b6..00000000000 --- a/pkg/tsdb/cloudwatch/get_metric_statistics_test.go +++ /dev/null @@ -1,187 +0,0 @@ -package cloudwatch - -import ( - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/components/null" - "github.com/grafana/grafana/pkg/components/simplejson" - . "github.com/smartystreets/goconvey/convey" -) - -func TestCloudWatchGetMetricStatistics(t *testing.T) { - Convey("CloudWatchGetMetricStatistics", t, func() { - - Convey("can parse cloudwatch json model", func() { - json := ` - { - "region": "us-east-1", - "namespace": "AWS/ApplicationELB", - "metricName": "TargetResponseTime", - "dimensions": { - "LoadBalancer": "lb", - "TargetGroup": "tg" - }, - "statistics": [ - "Average", - "Maximum", - "p50.00", - "p90.00" - ], - "period": "60", - "highResolution": false, - "alias": "{{metric}}_{{stat}}" - } - ` - modelJson, err := simplejson.NewJson([]byte(json)) - So(err, ShouldBeNil) - - res, err := parseQuery(modelJson) - So(err, ShouldBeNil) - So(res.Region, ShouldEqual, "us-east-1") - So(res.Namespace, ShouldEqual, "AWS/ApplicationELB") - So(res.MetricName, ShouldEqual, "TargetResponseTime") - So(len(res.Dimensions), ShouldEqual, 2) - So(*res.Dimensions[0].Name, ShouldEqual, "LoadBalancer") - So(*res.Dimensions[0].Value, ShouldEqual, "lb") - So(*res.Dimensions[1].Name, ShouldEqual, "TargetGroup") - So(*res.Dimensions[1].Value, ShouldEqual, "tg") - So(len(res.Statistics), ShouldEqual, 2) - So(*res.Statistics[0], ShouldEqual, "Average") - So(*res.Statistics[1], ShouldEqual, "Maximum") - So(len(res.ExtendedStatistics), ShouldEqual, 2) - So(*res.ExtendedStatistics[0], ShouldEqual, "p50.00") - So(*res.ExtendedStatistics[1], ShouldEqual, "p90.00") - So(res.Period, ShouldEqual, 60) - So(res.Alias, ShouldEqual, "{{metric}}_{{stat}}") - }) - - Convey("can parse cloudwatch response", func() { - timestamp := time.Unix(0, 0) - resp := &cloudwatch.GetMetricStatisticsOutput{ - Label: aws.String("TargetResponseTime"), - Datapoints: []*cloudwatch.Datapoint{ - { - Timestamp: aws.Time(timestamp), - Average: aws.Float64(10.0), - Maximum: aws.Float64(20.0), - ExtendedStatistics: map[string]*float64{ - "p50.00": aws.Float64(30.0), - "p90.00": aws.Float64(40.0), - }, - Unit: aws.String("Seconds"), - }, - }, - } - query := &CloudWatchQuery{ - Region: "us-east-1", - Namespace: "AWS/ApplicationELB", - MetricName: "TargetResponseTime", - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("LoadBalancer"), - Value: aws.String("lb"), - }, - { - Name: aws.String("TargetGroup"), - Value: aws.String("tg"), - }, - }, - Statistics: []*string{aws.String("Average"), aws.String("Maximum")}, - ExtendedStatistics: []*string{aws.String("p50.00"), aws.String("p90.00")}, - Period: 60, - Alias: "{{namespace}}_{{metric}}_{{stat}}", - } - - queryRes, err := parseResponse(resp, query) - So(err, ShouldBeNil) - So(queryRes.Series[0].Name, ShouldEqual, "AWS/ApplicationELB_TargetResponseTime_Average") - So(queryRes.Series[0].Tags["LoadBalancer"], ShouldEqual, "lb") - So(queryRes.Series[0].Tags["TargetGroup"], ShouldEqual, "tg") - So(queryRes.Series[0].Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) - So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) - So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) - So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) - So(queryRes.Meta.Get("unit").MustString(), ShouldEqual, "s") - }) - - Convey("terminate gap of data points", func() { - timestamp := time.Unix(0, 0) - resp := &cloudwatch.GetMetricStatisticsOutput{ - Label: aws.String("TargetResponseTime"), - Datapoints: []*cloudwatch.Datapoint{ - { - Timestamp: aws.Time(timestamp), - Average: aws.Float64(10.0), - Maximum: aws.Float64(20.0), - ExtendedStatistics: map[string]*float64{ - "p50.00": aws.Float64(30.0), - "p90.00": aws.Float64(40.0), - }, - Unit: aws.String("Seconds"), - }, - { - Timestamp: aws.Time(timestamp.Add(60 * time.Second)), - Average: aws.Float64(20.0), - Maximum: aws.Float64(30.0), - ExtendedStatistics: map[string]*float64{ - "p50.00": aws.Float64(40.0), - "p90.00": aws.Float64(50.0), - }, - Unit: aws.String("Seconds"), - }, - { - Timestamp: aws.Time(timestamp.Add(180 * time.Second)), - Average: aws.Float64(30.0), - Maximum: aws.Float64(40.0), - ExtendedStatistics: map[string]*float64{ - "p50.00": aws.Float64(50.0), - "p90.00": aws.Float64(60.0), - }, - Unit: aws.String("Seconds"), - }, - }, - } - query := &CloudWatchQuery{ - Region: "us-east-1", - Namespace: "AWS/ApplicationELB", - MetricName: "TargetResponseTime", - Dimensions: []*cloudwatch.Dimension{ - { - Name: aws.String("LoadBalancer"), - Value: aws.String("lb"), - }, - { - Name: aws.String("TargetGroup"), - Value: aws.String("tg"), - }, - }, - Statistics: []*string{aws.String("Average"), aws.String("Maximum")}, - ExtendedStatistics: []*string{aws.String("p50.00"), aws.String("p90.00")}, - Period: 60, - Alias: "{{namespace}}_{{metric}}_{{stat}}", - } - - queryRes, err := parseResponse(resp, query) - So(err, ShouldBeNil) - So(queryRes.Series[0].Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) - So(queryRes.Series[1].Points[0][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) - So(queryRes.Series[2].Points[0][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) - So(queryRes.Series[3].Points[0][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) - So(queryRes.Series[0].Points[1][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) - So(queryRes.Series[1].Points[1][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) - So(queryRes.Series[2].Points[1][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) - So(queryRes.Series[3].Points[1][0].String(), ShouldEqual, null.FloatFrom(50.0).String()) - So(queryRes.Series[0].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) - So(queryRes.Series[1].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) - So(queryRes.Series[2].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) - So(queryRes.Series[3].Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) - So(queryRes.Series[0].Points[3][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) - So(queryRes.Series[1].Points[3][0].String(), ShouldEqual, null.FloatFrom(40.0).String()) - So(queryRes.Series[2].Points[3][0].String(), ShouldEqual, null.FloatFrom(50.0).String()) - So(queryRes.Series[3].Points[3][0].String(), ShouldEqual, null.FloatFrom(60.0).String()) - }) - }) -} diff --git a/pkg/tsdb/cloudwatch/metric_data_input_builder.go b/pkg/tsdb/cloudwatch/metric_data_input_builder.go new file mode 100644 index 00000000000..cb8b5083f0c --- /dev/null +++ b/pkg/tsdb/cloudwatch/metric_data_input_builder.go @@ -0,0 +1,46 @@ +package cloudwatch + +import ( + "errors" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *CloudWatchExecutor) buildMetricDataInput(queryContext *tsdb.TsdbQuery, queries map[string]*cloudWatchQuery) (*cloudwatch.GetMetricDataInput, error) { + startTime, err := queryContext.TimeRange.ParseFrom() + if err != nil { + return nil, err + } + + endTime, err := queryContext.TimeRange.ParseTo() + if err != nil { + return nil, err + } + + if !startTime.Before(endTime) { + return nil, fmt.Errorf("Invalid time range: Start time must be before end time") + } + + metricDataInput := &cloudwatch.GetMetricDataInput{ + StartTime: aws.Time(startTime), + EndTime: aws.Time(endTime), + ScanBy: aws.String("TimestampAscending"), + } + for _, query := range queries { + // 1 minutes resolution metrics is stored for 15 days, 15 * 24 * 60 = 21600 + if query.HighResolution && (((endTime.Unix() - startTime.Unix()) / int64(query.Period)) > 21600) { + return nil, &queryError{errors.New("too long query period"), query.RefId} + } + + metricDataQuery, err := e.buildMetricDataQuery(query) + if err != nil { + return nil, &queryError{err, query.RefId} + } + metricDataInput.MetricDataQueries = append(metricDataInput.MetricDataQueries, metricDataQuery) + } + + return metricDataInput, nil +} diff --git a/pkg/tsdb/cloudwatch/metric_data_input_builder_test.go b/pkg/tsdb/cloudwatch/metric_data_input_builder_test.go new file mode 100644 index 00000000000..85f32c2b342 --- /dev/null +++ b/pkg/tsdb/cloudwatch/metric_data_input_builder_test.go @@ -0,0 +1,28 @@ +package cloudwatch + +import ( + "testing" + + "github.com/grafana/grafana/pkg/tsdb" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestMetricDataInputBuilder(t *testing.T) { + Convey("TestMetricDataInputBuilder", t, func() { + executor := &CloudWatchExecutor{} + query := make(map[string]*cloudWatchQuery) + + Convey("Time range is valid", func() { + Convey("End time before start time should result in error", func() { + _, err := executor.buildMetricDataInput(&tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")}, query) + So(err.Error(), ShouldEqual, "Invalid time range: Start time must be before end time") + }) + + Convey("End time equals start time should result in error", func() { + _, err := executor.buildMetricDataInput(&tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-1h")}, query) + So(err.Error(), ShouldEqual, "Invalid time range: Start time must be before end time") + }) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder.go b/pkg/tsdb/cloudwatch/metric_data_query_builder.go new file mode 100644 index 00000000000..2df063c7c52 --- /dev/null +++ b/pkg/tsdb/cloudwatch/metric_data_query_builder.go @@ -0,0 +1,139 @@ +package cloudwatch + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" +) + +func (e *CloudWatchExecutor) buildMetricDataQuery(query *cloudWatchQuery) (*cloudwatch.MetricDataQuery, error) { + mdq := &cloudwatch.MetricDataQuery{ + Id: aws.String(query.Id), + ReturnData: aws.Bool(query.ReturnData), + } + + if query.Expression != "" { + mdq.Expression = aws.String(query.Expression) + } else { + if query.isSearchExpression() { + mdq.Expression = aws.String(buildSearchExpression(query, query.Stats)) + } else { + mdq.MetricStat = &cloudwatch.MetricStat{ + Metric: &cloudwatch.Metric{ + Namespace: aws.String(query.Namespace), + MetricName: aws.String(query.MetricName), + Dimensions: make([]*cloudwatch.Dimension, 0), + }, + Period: aws.Int64(int64(query.Period)), + } + for key, values := range query.Dimensions { + mdq.MetricStat.Metric.Dimensions = append(mdq.MetricStat.Metric.Dimensions, + &cloudwatch.Dimension{ + Name: aws.String(key), + Value: aws.String(values[0]), + }) + } + mdq.MetricStat.Stat = aws.String(query.Stats) + } + } + + if mdq.Expression != nil { + query.UsedExpression = *mdq.Expression + } else { + query.UsedExpression = "" + } + + return mdq, nil +} + +func buildSearchExpression(query *cloudWatchQuery, stat string) string { + knownDimensions := make(map[string][]string) + dimensionNames := []string{} + dimensionNamesWithoutKnownValues := []string{} + + for key, values := range query.Dimensions { + dimensionNames = append(dimensionNames, key) + hasWildcard := false + for _, value := range values { + if value == "*" { + hasWildcard = true + break + } + } + if hasWildcard { + dimensionNamesWithoutKnownValues = append(dimensionNamesWithoutKnownValues, key) + } else { + knownDimensions[key] = values + } + } + + searchTerm := fmt.Sprintf(`MetricName="%s"`, query.MetricName) + keys := []string{} + for k := range knownDimensions { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + values := escape(knownDimensions[key]) + valueExpression := join(values, " OR ", `"`, `"`) + if len(knownDimensions[key]) > 1 { + valueExpression = fmt.Sprintf(`(%s)`, valueExpression) + } + keyFilter := fmt.Sprintf(`"%s"=%s`, key, valueExpression) + searchTerm = appendSearch(searchTerm, keyFilter) + } + + if query.MatchExact { + schema := query.Namespace + if len(dimensionNames) > 0 { + sort.Strings(dimensionNames) + schema += fmt.Sprintf(",%s", join(dimensionNames, ",", "", "")) + } + + return fmt.Sprintf("REMOVE_EMPTY(SEARCH('{%s} %s', '%s', %s))", schema, searchTerm, stat, strconv.Itoa(query.Period)) + } + + sort.Strings(dimensionNamesWithoutKnownValues) + searchTerm = appendSearch(searchTerm, join(dimensionNamesWithoutKnownValues, " ", `"`, `"`)) + return fmt.Sprintf(`REMOVE_EMPTY(SEARCH('Namespace="%s" %s', '%s', %s))`, query.Namespace, searchTerm, stat, strconv.Itoa(query.Period)) +} + +func escape(arr []string) []string { + result := []string{} + for _, value := range arr { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, ")", `\)`) + value = strings.ReplaceAll(value, "(", `\(`) + value = strings.ReplaceAll(value, `"`, `\"`) + result = append(result, value) + } + + return result +} + +func join(arr []string, delimiter string, valuePrefix string, valueSuffix string) string { + result := "" + for index, value := range arr { + result += valuePrefix + value + valueSuffix + if index+1 != len(arr) { + result += delimiter + } + } + + return result +} + +func appendSearch(target string, value string) string { + if value != "" { + if target == "" { + return value + } + return fmt.Sprintf("%v %v", target, value) + } + + return target +} diff --git a/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go new file mode 100644 index 00000000000..54619ad0df9 --- /dev/null +++ b/pkg/tsdb/cloudwatch/metric_data_query_builder_test.go @@ -0,0 +1,215 @@ +package cloudwatch + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestMetricDataQueryBuilder(t *testing.T) { + Convey("TestMetricDataQueryBuilder", t, func() { + Convey("buildSearchExpression", func() { + Convey("and query should be matched exact", func() { + matchExact := true + Convey("and query has three dimension values for a given dimension key", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('{AWS/EC2,LoadBalancer} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`) + }) + + Convey("and query has three dimension values for two given dimension keys", func() { + + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "i-456", "i-789"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('{AWS/EC2,InstanceId,LoadBalancer} MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`) + }) + + Convey("and no OR operator was added if a star was used for dimension value", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"*"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldNotContainSubstring, "OR") + }) + + Convey("and query has one dimension key with a * value", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"*"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('{AWS/EC2,LoadBalancer} MetricName="CPUUtilization"', 'Average', 300))`) + }) + + Convey("and query has three dimension values for two given dimension keys, and one value is a star", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "*", "i-789"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('{AWS/EC2,InstanceId,LoadBalancer} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`) + }) + }) + + Convey("and query should not be matched exact", func() { + matchExact := false + Convey("and query has three dimension values for a given dimension key", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`) + }) + + Convey("and query has three dimension values for two given dimension keys", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "i-456", "i-789"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`) + }) + + Convey("and query has one dimension key with a * value", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"*"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"', 'Average', 300))`) + }) + + Convey("and query has three dimension values for two given dimension keys, and one value is a star", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb1", "lb2", "lb3"}, + "InstanceId": {"i-123", "*", "i-789"}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: matchExact, + } + + res := buildSearchExpression(query, "Average") + So(res, ShouldEqual, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId"', 'Average', 300))`) + }) + + }) + }) + + Convey("and query has has invalid characters in dimension values", func() { + query := &cloudWatchQuery{ + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: map[string][]string{ + "lb1": {`lb\1\`}, + "lb2": {`)lb2`}, + "lb3": {`l(b3`}, + "lb4": {`lb4""`}, + "lb5": {`l\(b5"`}, + "lb6": {`l\\(b5"`}, + }, + Period: 300, + Identifier: "id1", + Expression: "", + MatchExact: true, + } + res := buildSearchExpression(query, "Average") + + Convey("it should escape backslash", func() { + So(res, ShouldContainSubstring, `"lb1"="lb\\1\\"`) + }) + + Convey("it should escape closing parenthesis", func() { + So(res, ShouldContainSubstring, `"lb2"="\)lb2"`) + }) + + Convey("it should escape open parenthesis", func() { + So(res, ShouldContainSubstring, `"lb3"="l\(b3"`) + }) + + Convey("it should escape double quotes", func() { + So(res, ShouldContainSubstring, `"lb6"="l\\\\\(b5\""`) + }) + + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/metric_find_query.go b/pkg/tsdb/cloudwatch/metric_find_query.go index a5abace082c..8a4d41d3d5a 100644 --- a/pkg/tsdb/cloudwatch/metric_find_query.go +++ b/pkg/tsdb/cloudwatch/metric_find_query.go @@ -637,10 +637,13 @@ func (e *CloudWatchExecutor) cloudwatchListMetrics(region string, namespace stri params := &cloudwatch.ListMetricsInput{ Namespace: aws.String(namespace), - MetricName: aws.String(metricName), Dimensions: dimensions, } + if metricName != "" { + params.MetricName = aws.String(metricName) + } + var resp cloudwatch.ListMetricsOutput err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool { diff --git a/pkg/tsdb/cloudwatch/query_transformer.go b/pkg/tsdb/cloudwatch/query_transformer.go new file mode 100644 index 00000000000..2ba5c729447 --- /dev/null +++ b/pkg/tsdb/cloudwatch/query_transformer.go @@ -0,0 +1,103 @@ +package cloudwatch + +import ( + "fmt" + "sort" + "strings" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" +) + +// returns a map of queries with query id as key. In the case a q request query +// has more than one statistic defined, one cloudwatchQuery will be created for each statistic. +// If the query doesn't have an Id defined by the user, we'll give it an with format `query[RefId]`. In the case +// the incoming query had more than one stat, it will ge an id like `query[RefId]_[StatName]`, eg queryC_Average +func (e *CloudWatchExecutor) transformRequestQueriesToCloudWatchQueries(requestQueries []*requestQuery) (map[string]*cloudWatchQuery, error) { + cloudwatchQueries := make(map[string]*cloudWatchQuery) + for _, requestQuery := range requestQueries { + for _, stat := range requestQuery.Statistics { + id := requestQuery.Id + if id == "" { + id = fmt.Sprintf("query%s", requestQuery.RefId) + } + if len(requestQuery.Statistics) > 1 { + id = fmt.Sprintf("%s_%v", id, strings.ReplaceAll(*stat, ".", "_")) + } + + query := &cloudWatchQuery{ + Id: id, + RefId: requestQuery.RefId, + Region: requestQuery.Region, + Namespace: requestQuery.Namespace, + MetricName: requestQuery.MetricName, + Dimensions: requestQuery.Dimensions, + Stats: *stat, + Period: requestQuery.Period, + Alias: requestQuery.Alias, + Expression: requestQuery.Expression, + ReturnData: requestQuery.ReturnData, + HighResolution: requestQuery.HighResolution, + MatchExact: requestQuery.MatchExact, + } + + if _, ok := cloudwatchQueries[id]; ok { + return nil, fmt.Errorf("Error in query %s. Query id %s is not unique", query.RefId, query.Id) + } + + cloudwatchQueries[id] = query + } + } + + return cloudwatchQueries, nil +} + +func (e *CloudWatchExecutor) transformQueryResponseToQueryResult(cloudwatchResponses []*cloudwatchResponse) map[string]*tsdb.QueryResult { + results := make(map[string]*tsdb.QueryResult) + responsesByRefID := make(map[string][]*cloudwatchResponse) + + for _, res := range cloudwatchResponses { + if _, ok := responsesByRefID[res.RefId]; ok { + responsesByRefID[res.RefId] = append(responsesByRefID[res.RefId], res) + } else { + responsesByRefID[res.RefId] = []*cloudwatchResponse{res} + } + } + + for refID, responses := range responsesByRefID { + queryResult := tsdb.NewQueryResult() + queryResult.RefId = refID + queryResult.Meta = simplejson.New() + queryResult.Series = tsdb.TimeSeriesSlice{} + timeSeries := make(tsdb.TimeSeriesSlice, 0) + + requestExceededMaxLimit := false + queryMeta := []struct { + Expression, ID string + }{} + + for _, response := range responses { + timeSeries = append(timeSeries, *response.series...) + requestExceededMaxLimit = requestExceededMaxLimit || response.RequestExceededMaxLimit + queryMeta = append(queryMeta, struct { + Expression, ID string + }{ + Expression: response.Expression, + ID: response.Id, + }) + } + + sort.Slice(timeSeries, func(i, j int) bool { + return timeSeries[i].Name < timeSeries[j].Name + }) + + if requestExceededMaxLimit { + queryResult.ErrorString = "Cloudwatch GetMetricData error: Maximum number of allowed metrics exceeded. Your search may have been limited." + } + queryResult.Series = append(queryResult.Series, timeSeries...) + queryResult.Meta.Set("gmdMeta", queryMeta) + results[refID] = queryResult + } + + return results +} diff --git a/pkg/tsdb/cloudwatch/query_transformer_test.go b/pkg/tsdb/cloudwatch/query_transformer_test.go new file mode 100644 index 00000000000..cef9b6366d4 --- /dev/null +++ b/pkg/tsdb/cloudwatch/query_transformer_test.go @@ -0,0 +1,167 @@ +package cloudwatch + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + . "github.com/smartystreets/goconvey/convey" +) + +func TestQueryTransformer(t *testing.T) { + Convey("TestQueryTransformer", t, func() { + Convey("when transforming queries", func() { + + executor := &CloudWatchExecutor{} + Convey("one cloudwatchQuery is generated when its request query has one stat", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average"}), + Period: 600, + Id: "", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 1) + }) + + Convey("two cloudwatchQuery is generated when there's two stats", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average", "Sum"}), + Period: 600, + Id: "", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 2) + }) + Convey("and id is given by user", func() { + Convey("that id will be used in the cloudwatch query", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average"}), + Period: 600, + Id: "myid", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 1) + So(res, ShouldContainKey, "myid") + }) + }) + + Convey("and id is not given by user", func() { + Convey("id will be generated based on ref id if query only has one stat", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average"}), + Period: 600, + Id: "", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 1) + So(res, ShouldContainKey, "queryD") + }) + + Convey("id will be generated based on ref and stat name if query has two stats", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average", "Sum"}), + Period: 600, + Id: "", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 2) + So(res, ShouldContainKey, "queryD_Sum") + So(res, ShouldContainKey, "queryD_Average") + }) + }) + + Convey("dot should be removed when query has more than one stat and one of them is a percentile", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average", "p46.32"}), + Period: 600, + Id: "", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(err, ShouldBeNil) + So(len(res), ShouldEqual, 2) + So(res, ShouldContainKey, "queryD_p46_32") + }) + + Convey("should return an error if two queries have the same id", func() { + requestQueries := []*requestQuery{ + { + RefId: "D", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average", "p46.32"}), + Period: 600, + Id: "myId", + HighResolution: false, + }, + { + RefId: "E", + Region: "us-east-1", + Namespace: "ec2", + MetricName: "CPUUtilization", + Statistics: aws.StringSlice([]string{"Average", "p46.32"}), + Period: 600, + Id: "myId", + HighResolution: false, + }, + } + + res, err := executor.transformRequestQueriesToCloudWatchQueries(requestQueries) + So(res, ShouldBeNil) + So(err, ShouldNotBeNil) + }) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/request_parser.go b/pkg/tsdb/cloudwatch/request_parser.go new file mode 100644 index 00000000000..5da32d48231 --- /dev/null +++ b/pkg/tsdb/cloudwatch/request_parser.go @@ -0,0 +1,162 @@ +package cloudwatch + +import ( + "errors" + "regexp" + "sort" + "strconv" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/tsdb" +) + +// Parses the json queries and returns a requestQuery. The requstQuery has a 1 to 1 mapping to a query editor row +func (e *CloudWatchExecutor) parseQueries(queryContext *tsdb.TsdbQuery) (map[string][]*requestQuery, error) { + requestQueries := make(map[string][]*requestQuery) + + for i, model := range queryContext.Queries { + queryType := model.Model.Get("type").MustString() + if queryType != "timeSeriesQuery" && queryType != "" { + continue + } + + RefID := queryContext.Queries[i].RefId + query, err := parseRequestQuery(queryContext.Queries[i].Model, RefID) + if err != nil { + return nil, &queryError{err, RefID} + } + if _, exist := requestQueries[query.Region]; !exist { + requestQueries[query.Region] = make([]*requestQuery, 0) + } + requestQueries[query.Region] = append(requestQueries[query.Region], query) + } + + return requestQueries, nil +} + +func parseRequestQuery(model *simplejson.Json, refId string) (*requestQuery, error) { + region, err := model.Get("region").String() + if err != nil { + return nil, err + } + + namespace, err := model.Get("namespace").String() + if err != nil { + return nil, err + } + + metricName, err := model.Get("metricName").String() + if err != nil { + return nil, err + } + + dimensions, err := parseDimensions(model) + if err != nil { + return nil, err + } + + statistics, err := parseStatistics(model) + if err != nil { + return nil, err + } + + p := model.Get("period").MustString("") + if p == "" { + if namespace == "AWS/EC2" { + p = "300" + } else { + p = "60" + } + } + + var period int + if regexp.MustCompile(`^\d+$`).Match([]byte(p)) { + period, err = strconv.Atoi(p) + if err != nil { + return nil, err + } + } else { + d, err := time.ParseDuration(p) + if err != nil { + return nil, err + } + period = int(d.Seconds()) + } + + id := model.Get("id").MustString("") + expression := model.Get("expression").MustString("") + alias := model.Get("alias").MustString() + returnData := !model.Get("hide").MustBool(false) + queryType := model.Get("type").MustString() + if queryType == "" { + // If no type is provided we assume we are called by alerting service, which requires to return data! + // Note, this is sort of a hack, but the official Grafana interfaces do not carry the information + // who (which service) called the TsdbQueryEndpoint.Query(...) function. + returnData = true + } + + highResolution := model.Get("highResolution").MustBool(false) + matchExact := model.Get("matchExact").MustBool(true) + + return &requestQuery{ + RefId: refId, + Region: region, + Namespace: namespace, + MetricName: metricName, + Dimensions: dimensions, + Statistics: aws.StringSlice(statistics), + Period: period, + Alias: alias, + Id: id, + Expression: expression, + ReturnData: returnData, + HighResolution: highResolution, + MatchExact: matchExact, + }, nil +} + +func parseStatistics(model *simplejson.Json) ([]string, error) { + var statistics []string + + for _, s := range model.Get("statistics").MustArray() { + statistics = append(statistics, s.(string)) + } + + return statistics, nil +} + +func parseDimensions(model *simplejson.Json) (map[string][]string, error) { + parsedDimensions := make(map[string][]string) + for k, v := range model.Get("dimensions").MustMap() { + // This is for backwards compatibility. Before 6.5 dimensions values were stored as strings and not arrays + if value, ok := v.(string); ok { + parsedDimensions[k] = []string{value} + } else if values, ok := v.([]interface{}); ok { + for _, value := range values { + parsedDimensions[k] = append(parsedDimensions[k], value.(string)) + } + } else { + return nil, errors.New("failed to parse dimensions") + } + } + + sortedDimensions := sortDimensions(parsedDimensions) + + return sortedDimensions, nil +} + +func sortDimensions(dimensions map[string][]string) map[string][]string { + sortedDimensions := make(map[string][]string) + var keys []string + for k := range dimensions { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + sortedDimensions[k] = dimensions[k] + } + return sortedDimensions +} diff --git a/pkg/tsdb/cloudwatch/request_parser_test.go b/pkg/tsdb/cloudwatch/request_parser_test.go new file mode 100644 index 00000000000..78692aa4329 --- /dev/null +++ b/pkg/tsdb/cloudwatch/request_parser_test.go @@ -0,0 +1,87 @@ +package cloudwatch + +import ( + "testing" + + "github.com/grafana/grafana/pkg/components/simplejson" + . "github.com/smartystreets/goconvey/convey" +) + +func TestRequestParser(t *testing.T) { + Convey("TestRequestParser", t, func() { + Convey("when parsing query editor row json", func() { + Convey("using new dimensions structure", func() { + query := simplejson.NewFromAny(map[string]interface{}{ + "refId": "ref1", + "region": "us-east-1", + "namespace": "ec2", + "metricName": "CPUUtilization", + "id": "", + "expression": "", + "dimensions": map[string]interface{}{ + "InstanceId": []interface{}{"test"}, + "InstanceType": []interface{}{"test2", "test3"}, + }, + "statistics": []interface{}{"Average"}, + "period": "600", + "hide": false, + "highResolution": false, + }) + + res, err := parseRequestQuery(query, "ref1") + So(err, ShouldBeNil) + So(res.Region, ShouldEqual, "us-east-1") + So(res.RefId, ShouldEqual, "ref1") + So(res.Namespace, ShouldEqual, "ec2") + So(res.MetricName, ShouldEqual, "CPUUtilization") + So(res.Id, ShouldEqual, "") + So(res.Expression, ShouldEqual, "") + So(res.Period, ShouldEqual, 600) + So(res.ReturnData, ShouldEqual, true) + So(res.HighResolution, ShouldEqual, false) + So(len(res.Dimensions), ShouldEqual, 2) + So(len(res.Dimensions["InstanceId"]), ShouldEqual, 1) + So(len(res.Dimensions["InstanceType"]), ShouldEqual, 2) + So(res.Dimensions["InstanceType"][1], ShouldEqual, "test3") + So(len(res.Statistics), ShouldEqual, 1) + So(*res.Statistics[0], ShouldEqual, "Average") + }) + + Convey("using old dimensions structure (backwards compatibility)", func() { + query := simplejson.NewFromAny(map[string]interface{}{ + "refId": "ref1", + "region": "us-east-1", + "namespace": "ec2", + "metricName": "CPUUtilization", + "id": "", + "expression": "", + "dimensions": map[string]interface{}{ + "InstanceId": "test", + "InstanceType": "test2", + }, + "statistics": []interface{}{"Average"}, + "period": "600", + "hide": false, + "highResolution": false, + }) + + res, err := parseRequestQuery(query, "ref1") + So(err, ShouldBeNil) + So(res.Region, ShouldEqual, "us-east-1") + So(res.RefId, ShouldEqual, "ref1") + So(res.Namespace, ShouldEqual, "ec2") + So(res.MetricName, ShouldEqual, "CPUUtilization") + So(res.Id, ShouldEqual, "") + So(res.Expression, ShouldEqual, "") + So(res.Period, ShouldEqual, 600) + So(res.ReturnData, ShouldEqual, true) + So(res.HighResolution, ShouldEqual, false) + So(len(res.Dimensions), ShouldEqual, 2) + So(len(res.Dimensions["InstanceId"]), ShouldEqual, 1) + So(len(res.Dimensions["InstanceType"]), ShouldEqual, 1) + So(res.Dimensions["InstanceType"][0], ShouldEqual, "test2") + So(*res.Statistics[0], ShouldEqual, "Average") + }) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/response_parser.go b/pkg/tsdb/cloudwatch/response_parser.go new file mode 100644 index 00000000000..6a91b0395fe --- /dev/null +++ b/pkg/tsdb/cloudwatch/response_parser.go @@ -0,0 +1,155 @@ +package cloudwatch + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/components/null" + "github.com/grafana/grafana/pkg/tsdb" +) + +func (e *CloudWatchExecutor) parseResponse(metricDataOutputs []*cloudwatch.GetMetricDataOutput, queries map[string]*cloudWatchQuery) ([]*cloudwatchResponse, error) { + mdr := make(map[string]map[string]*cloudwatch.MetricDataResult) + + for _, mdo := range metricDataOutputs { + requestExceededMaxLimit := false + for _, message := range mdo.Messages { + if *message.Code == "MaxMetricsExceeded" { + requestExceededMaxLimit = true + } + } + + for _, r := range mdo.MetricDataResults { + if _, exists := mdr[*r.Id]; !exists { + mdr[*r.Id] = make(map[string]*cloudwatch.MetricDataResult) + mdr[*r.Id][*r.Label] = r + } else if _, exists := mdr[*r.Id][*r.Label]; !exists { + mdr[*r.Id][*r.Label] = r + } else { + mdr[*r.Id][*r.Label].Timestamps = append(mdr[*r.Id][*r.Label].Timestamps, r.Timestamps...) + mdr[*r.Id][*r.Label].Values = append(mdr[*r.Id][*r.Label].Values, r.Values...) + } + queries[*r.Id].RequestExceededMaxLimit = requestExceededMaxLimit + } + } + + cloudWatchResponses := make([]*cloudwatchResponse, 0) + for id, lr := range mdr { + response := &cloudwatchResponse{} + series, err := parseGetMetricDataTimeSeries(lr, queries[id]) + if err != nil { + return cloudWatchResponses, err + } + + response.series = series + response.Expression = queries[id].UsedExpression + response.RefId = queries[id].RefId + response.Id = queries[id].Id + response.RequestExceededMaxLimit = queries[id].RequestExceededMaxLimit + + cloudWatchResponses = append(cloudWatchResponses, response) + } + + return cloudWatchResponses, nil +} + +func parseGetMetricDataTimeSeries(metricDataResults map[string]*cloudwatch.MetricDataResult, query *cloudWatchQuery) (*tsdb.TimeSeriesSlice, error) { + result := tsdb.TimeSeriesSlice{} + for label, metricDataResult := range metricDataResults { + if *metricDataResult.StatusCode != "Complete" { + return nil, fmt.Errorf("too many datapoint requested in query %s. Please try to reduce the time range", query.RefId) + } + + for _, message := range metricDataResult.Messages { + if *message.Code == "ArithmeticError" { + return nil, fmt.Errorf("ArithmeticError in query %s: %s", query.RefId, *message.Value) + } + } + + series := tsdb.TimeSeries{ + Tags: make(map[string]string), + Points: make([]tsdb.TimePoint, 0), + } + + for key, values := range query.Dimensions { + if len(values) == 1 && values[0] != "*" { + series.Tags[key] = values[0] + } else { + for _, value := range values { + if value == label || value == "*" { + series.Tags[key] = label + } + } + } + } + + series.Name = formatAlias(query, query.Stats, series.Tags, label) + + for j, t := range metricDataResult.Timestamps { + if j > 0 { + expectedTimestamp := metricDataResult.Timestamps[j-1].Add(time.Duration(query.Period) * time.Second) + if expectedTimestamp.Before(*t) { + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFromPtr(nil), float64(expectedTimestamp.Unix()*1000))) + } + } + series.Points = append(series.Points, tsdb.NewTimePoint(null.FloatFrom(*metricDataResult.Values[j]), float64((*t).Unix())*1000)) + } + result = append(result, &series) + } + return &result, nil +} + +func formatAlias(query *cloudWatchQuery, stat string, dimensions map[string]string, label string) string { + region := query.Region + namespace := query.Namespace + metricName := query.MetricName + period := strconv.Itoa(query.Period) + + if query.isUserDefinedSearchExpression() { + pIndex := strings.LastIndex(query.Expression, ",") + period = strings.Trim(query.Expression[pIndex+1:], " )") + sIndex := strings.LastIndex(query.Expression[:pIndex], ",") + stat = strings.Trim(query.Expression[sIndex+1:pIndex], " '") + } + + if len(query.Alias) == 0 && query.isMathExpression() { + return query.Id + } + + if len(query.Alias) == 0 && query.isInferredSearchExpression() { + return label + } + + data := map[string]string{} + data["region"] = region + data["namespace"] = namespace + data["metric"] = metricName + data["stat"] = stat + data["period"] = period + if len(label) != 0 { + data["label"] = label + } + for k, v := range dimensions { + data[k] = v + } + + result := aliasFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte { + labelName := strings.Replace(string(in), "{{", "", 1) + labelName = strings.Replace(labelName, "}}", "", 1) + labelName = strings.TrimSpace(labelName) + if val, exists := data[labelName]; exists { + return []byte(val) + } + + return in + }) + + if string(result) == "" { + return metricName + "_" + stat + } + + return string(result) +} diff --git a/pkg/tsdb/cloudwatch/response_parser_test.go b/pkg/tsdb/cloudwatch/response_parser_test.go new file mode 100644 index 00000000000..e5921f1dce5 --- /dev/null +++ b/pkg/tsdb/cloudwatch/response_parser_test.go @@ -0,0 +1,61 @@ +package cloudwatch + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/components/null" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCloudWatchResponseParser(t *testing.T) { + Convey("TestCloudWatchResponseParser", t, func() { + + Convey("can parse cloudwatch response", func() { + timestamp := time.Unix(0, 0) + resp := map[string]*cloudwatch.MetricDataResult{ + "lb": { + Id: aws.String("id1"), + Label: aws.String("lb"), + Timestamps: []*time.Time{ + aws.Time(timestamp), + aws.Time(timestamp.Add(60 * time.Second)), + aws.Time(timestamp.Add(180 * time.Second)), + }, + Values: []*float64{ + aws.Float64(10), + aws.Float64(20), + aws.Float64(30), + }, + StatusCode: aws.String("Complete"), + }, + } + + query := &cloudWatchQuery{ + RefId: "refId1", + Region: "us-east-1", + Namespace: "AWS/ApplicationELB", + MetricName: "TargetResponseTime", + Dimensions: map[string][]string{ + "LoadBalancer": {"lb"}, + "TargetGroup": {"tg"}, + }, + Stats: "Average", + Period: 60, + Alias: "{{namespace}}_{{metric}}_{{stat}}", + } + series, err := parseGetMetricDataTimeSeries(resp, query) + timeSeries := (*series)[0] + + So(err, ShouldBeNil) + So(timeSeries.Name, ShouldEqual, "AWS/ApplicationELB_TargetResponseTime_Average") + So(timeSeries.Tags["LoadBalancer"], ShouldEqual, "lb") + So(timeSeries.Points[0][0].String(), ShouldEqual, null.FloatFrom(10.0).String()) + So(timeSeries.Points[1][0].String(), ShouldEqual, null.FloatFrom(20.0).String()) + So(timeSeries.Points[2][0].String(), ShouldEqual, null.FloatFromPtr(nil).String()) + So(timeSeries.Points[3][0].String(), ShouldEqual, null.FloatFrom(30.0).String()) + }) + }) +} diff --git a/pkg/tsdb/cloudwatch/time_series_query.go b/pkg/tsdb/cloudwatch/time_series_query.go new file mode 100644 index 00000000000..016fe27c0f3 --- /dev/null +++ b/pkg/tsdb/cloudwatch/time_series_query.go @@ -0,0 +1,101 @@ +package cloudwatch + +import ( + "context" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/tsdb" + "golang.org/x/sync/errgroup" +) + +func (e *CloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, queryContext *tsdb.TsdbQuery) (*tsdb.Response, error) { + results := &tsdb.Response{ + Results: make(map[string]*tsdb.QueryResult), + } + + requestQueriesByRegion, err := e.parseQueries(queryContext) + if err != nil { + return results, err + } + resultChan := make(chan *tsdb.QueryResult, len(queryContext.Queries)) + eg, ectx := errgroup.WithContext(ctx) + + if len(requestQueriesByRegion) > 0 { + for r, q := range requestQueriesByRegion { + requestQueries := q + region := r + eg.Go(func() error { + defer func() { + if err := recover(); err != nil { + plog.Error("Execute Get Metric Data Query Panic", "error", err, "stack", log.Stack(1)) + if theErr, ok := err.(error); ok { + resultChan <- &tsdb.QueryResult{ + Error: theErr, + } + } + } + }() + + client, err := e.getClient(region) + if err != nil { + return err + } + + queries, err := e.transformRequestQueriesToCloudWatchQueries(requestQueries) + if err != nil { + for _, query := range requestQueries { + resultChan <- &tsdb.QueryResult{ + RefId: query.RefId, + Error: err, + } + } + return nil + } + + metricDataInput, err := e.buildMetricDataInput(queryContext, queries) + if err != nil { + return err + } + + cloudwatchResponses := make([]*cloudwatchResponse, 0) + mdo, err := e.executeRequest(ectx, client, metricDataInput) + if err != nil { + for _, query := range requestQueries { + resultChan <- &tsdb.QueryResult{ + RefId: query.RefId, + Error: err, + } + } + return nil + } + + responses, err := e.parseResponse(mdo, queries) + if err != nil { + for _, query := range requestQueries { + resultChan <- &tsdb.QueryResult{ + RefId: query.RefId, + Error: err, + } + } + return nil + } + cloudwatchResponses = append(cloudwatchResponses, responses...) + res := e.transformQueryResponseToQueryResult(cloudwatchResponses) + for _, queryRes := range res { + resultChan <- queryRes + } + return nil + }) + } + } + + if err := eg.Wait(); err != nil { + return nil, err + } + close(resultChan) + for result := range resultChan { + results.Results[result.RefId] = result + } + + return results, nil +} diff --git a/pkg/tsdb/cloudwatch/types.go b/pkg/tsdb/cloudwatch/types.go index 1225fb9b31b..c225cdd35e9 100644 --- a/pkg/tsdb/cloudwatch/types.go +++ b/pkg/tsdb/cloudwatch/types.go @@ -1,21 +1,49 @@ package cloudwatch import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/grafana/grafana/pkg/tsdb" ) -type CloudWatchQuery struct { +type cloudWatchClient interface { + GetMetricDataWithContext(ctx aws.Context, input *cloudwatch.GetMetricDataInput, opts ...request.Option) (*cloudwatch.GetMetricDataOutput, error) +} + +type requestQuery struct { RefId string Region string + Id string Namespace string MetricName string - Dimensions []*cloudwatch.Dimension Statistics []*string + QueryType string + Expression string + ReturnData bool + Dimensions map[string][]string ExtendedStatistics []*string Period int Alias string - Id string - Expression string - ReturnData bool HighResolution bool + MatchExact bool +} + +type cloudwatchResponse struct { + series *tsdb.TimeSeriesSlice + Id string + RefId string + Expression string + RequestExceededMaxLimit bool +} + +type queryError struct { + err error + RefID string +} + +func (e *queryError) Error() string { + return fmt.Sprintf("Error parsing query %s, %s", e.RefID, e.err) } diff --git a/public/app/core/components/AppNotifications/AppNotificationItem.tsx b/public/app/core/components/AppNotifications/AppNotificationItem.tsx index 86492801057..dc41260d7a0 100644 --- a/public/app/core/components/AppNotifications/AppNotificationItem.tsx +++ b/public/app/core/components/AppNotifications/AppNotificationItem.tsx @@ -26,7 +26,7 @@ export default class AppNotificationItem extends Component { onClearNotification(appNotification.id)} /> ); diff --git a/public/app/core/copy/appNotification.ts b/public/app/core/copy/appNotification.ts index 92cf7f8e118..e02f42e8537 100644 --- a/public/app/core/copy/appNotification.ts +++ b/public/app/core/copy/appNotification.ts @@ -32,12 +32,13 @@ export const createSuccessNotification = (title: string, text = ''): AppNotifica id: Date.now(), }); -export const createErrorNotification = (title: string, text = ''): AppNotification => { +export const createErrorNotification = (title: string, text = '', component?: React.ReactElement): AppNotification => { return { ...defaultErrorNotification, - title: title, text: getMessageFromError(text), + title, id: Date.now(), + component, }; }; diff --git a/public/app/plugins/datasource/cloudwatch/components/Alias.test.tsx b/public/app/plugins/datasource/cloudwatch/components/Alias.test.tsx new file mode 100644 index 00000000000..3b190c4ba9c --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Alias.test.tsx @@ -0,0 +1,10 @@ +import React from 'react'; +import renderer from 'react-test-renderer'; +import { Alias } from './Alias'; + +describe('Alias', () => { + it('should render component', () => { + const tree = renderer.create( {}} />).toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/Alias.tsx b/public/app/plugins/datasource/cloudwatch/components/Alias.tsx new file mode 100644 index 00000000000..3115783c570 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Alias.tsx @@ -0,0 +1,21 @@ +import React, { FunctionComponent, useState } from 'react'; +import { debounce } from 'lodash'; +import { Input } from '@grafana/ui'; + +export interface Props { + onChange: (alias: any) => void; + value: string; +} + +export const Alias: FunctionComponent = ({ value = '', onChange }) => { + const [alias, setAlias] = useState(value); + + const propagateOnChange = debounce(onChange, 1500); + + onChange = (e: any) => { + setAlias(e.target.value); + propagateOnChange(e.target.value); + }; + + return ; +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.test.tsx b/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.test.tsx new file mode 100644 index 00000000000..560035a1a91 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.test.tsx @@ -0,0 +1,106 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import ConfigEditor, { Props } from './ConfigEditor'; + +jest.mock('app/features/plugins/datasource_srv', () => ({ + getDatasourceSrv: () => ({ + loadDatasource: jest.fn().mockImplementation(() => + Promise.resolve({ + getRegions: jest.fn().mockReturnValue([ + { + label: 'ap-east-1', + value: 'ap-east-1', + }, + ]), + }) + ), + }), +})); + +const setup = (propOverrides?: object) => { + const props: Props = { + options: { + id: 1, + orgId: 1, + typeLogoUrl: '', + name: 'CloudWatch', + access: 'proxy', + url: '', + database: '', + type: 'cloudwatch', + user: '', + password: '', + basicAuth: false, + basicAuthPassword: '', + basicAuthUser: '', + isDefault: true, + readOnly: false, + withCredentials: false, + secureJsonFields: { + accessKey: false, + secretKey: false, + }, + jsonData: { + assumeRoleArn: '', + database: '', + customMetricsNamespaces: '', + authType: 'keys', + defaultRegion: 'us-east-2', + timeField: '@timestamp', + }, + secureJsonData: { + secretKey: '', + accessKey: '', + }, + }, + onOptionsChange: jest.fn(), + }; + + Object.assign(props, propOverrides); + + return shallow(); +}; + +describe('Render', () => { + it('should render component', () => { + const wrapper = setup(); + + expect(wrapper).toMatchSnapshot(); + }); + + it('should disable access key id field', () => { + const wrapper = setup({ + secureJsonFields: { + secretKey: true, + }, + }); + expect(wrapper).toMatchSnapshot(); + }); + + it('should should show credentials profile name field', () => { + const wrapper = setup({ + jsonData: { + authType: 'credentials', + }, + }); + expect(wrapper).toMatchSnapshot(); + }); + + it('should should show access key and secret access key fields', () => { + const wrapper = setup({ + jsonData: { + authType: 'keys', + }, + }); + expect(wrapper).toMatchSnapshot(); + }); + + it('should should show arn role field', () => { + const wrapper = setup({ + jsonData: { + authType: 'arn', + }, + }); + expect(wrapper).toMatchSnapshot(); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.tsx new file mode 100644 index 00000000000..2a443a0af0a --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/ConfigEditor.tsx @@ -0,0 +1,390 @@ +import React, { PureComponent, ChangeEvent } from 'react'; +import { FormLabel, Select, Input, Button } from '@grafana/ui'; +import { DataSourcePluginOptionsEditorProps, DataSourceSettings } from '@grafana/data'; +import { SelectableValue } from '@grafana/data'; +import { getDatasourceSrv } from 'app/features/plugins/datasource_srv'; +import CloudWatchDatasource from '../datasource'; +import { CloudWatchJsonData, CloudWatchSecureJsonData } from '../types'; + +export type Props = DataSourcePluginOptionsEditorProps; + +type CloudwatchSettings = DataSourceSettings; + +export interface State { + config: CloudwatchSettings; + authProviderOptions: SelectableValue[]; + regions: SelectableValue[]; +} + +export class ConfigEditor extends PureComponent { + constructor(props: Props) { + super(props); + + const { options } = this.props; + + this.state = { + config: ConfigEditor.defaults(options), + authProviderOptions: [ + { label: 'Access & secret key', value: 'keys' }, + { label: 'Credentials file', value: 'credentials' }, + { label: 'ARN', value: 'arn' }, + ], + regions: [], + }; + + this.updateDatasource(this.state.config); + } + + static getDerivedStateFromProps(props: Props, state: State) { + return { + ...state, + config: ConfigEditor.defaults(props.options), + }; + } + + static defaults = (options: any) => { + options.jsonData.authType = options.jsonData.authType || 'credentials'; + options.jsonData.timeField = options.jsonData.timeField || '@timestamp'; + + if (!options.hasOwnProperty('secureJsonData')) { + options.secureJsonData = {}; + } + + if (!options.hasOwnProperty('jsonData')) { + options.jsonData = {}; + } + + if (!options.hasOwnProperty('secureJsonFields')) { + options.secureJsonFields = {}; + } + + return options; + }; + + async componentDidMount() { + this.loadRegions(); + } + + loadRegions() { + getDatasourceSrv() + .loadDatasource(this.state.config.name) + .then((ds: CloudWatchDatasource) => { + return ds.getRegions(); + }) + .then( + (regions: any) => { + this.setState({ + regions: regions.map((region: any) => { + return { + value: region.value, + label: region.text, + }; + }), + }); + }, + (err: any) => { + const regions = [ + 'ap-east-1', + 'ap-northeast-1', + 'ap-northeast-2', + 'ap-northeast-3', + 'ap-south-1', + 'ap-southeast-1', + 'ap-southeast-2', + 'ca-central-1', + 'cn-north-1', + 'cn-northwest-1', + 'eu-central-1', + 'eu-north-1', + 'eu-west-1', + 'eu-west-2', + 'eu-west-3', + 'me-south-1', + 'sa-east-1', + 'us-east-1', + 'us-east-2', + 'us-gov-east-1', + 'us-gov-west-1', + 'us-iso-east-1', + 'us-isob-east-1', + 'us-west-1', + 'us-west-2', + ]; + + this.setState({ + regions: regions.map((region: string) => { + return { + value: region, + label: region, + }; + }), + }); + + // expected to fail when creating new datasource + // console.error('failed to get latest regions', err); + } + ); + } + + updateDatasource = async (config: any) => { + for (const j in config.jsonData) { + if (config.jsonData[j].length === 0) { + delete config.jsonData[j]; + } + } + + for (const k in config.secureJsonData) { + if (config.secureJsonData[k].length === 0) { + delete config.secureJsonData[k]; + } + } + + this.props.onOptionsChange({ + ...config, + }); + }; + + onAuthProviderChange = (authType: SelectableValue) => { + this.updateDatasource({ + ...this.state.config, + jsonData: { + ...this.state.config.jsonData, + authType: authType.value, + }, + }); + }; + + onRegionChange = (defaultRegion: SelectableValue) => { + this.updateDatasource({ + ...this.state.config, + jsonData: { + ...this.state.config.jsonData, + defaultRegion: defaultRegion.value, + }, + }); + }; + + onResetAccessKey = () => { + this.updateDatasource({ + ...this.state.config, + secureJsonFields: { + ...this.state.config.secureJsonFields, + accessKey: false, + }, + }); + }; + + onAccessKeyChange = (accessKey: string) => { + this.updateDatasource({ + ...this.state.config, + secureJsonData: { + ...this.state.config.secureJsonData, + accessKey, + }, + }); + }; + + onResetSecretKey = () => { + this.updateDatasource({ + ...this.state.config, + secureJsonFields: { + ...this.state.config.secureJsonFields, + secretKey: false, + }, + }); + }; + + onSecretKeyChange = (secretKey: string) => { + this.updateDatasource({ + ...this.state.config, + secureJsonData: { + ...this.state.config.secureJsonData, + secretKey, + }, + }); + }; + + onCredentialProfileNameChange = (database: string) => { + this.updateDatasource({ + ...this.state.config, + database, + }); + }; + + onArnAssumeRoleChange = (assumeRoleArn: string) => { + this.updateDatasource({ + ...this.state.config, + jsonData: { + ...this.state.config.jsonData, + assumeRoleArn, + }, + }); + }; + + onCustomMetricsNamespacesChange = (customMetricsNamespaces: string) => { + this.updateDatasource({ + ...this.state.config, + jsonData: { + ...this.state.config.jsonData, + customMetricsNamespaces, + }, + }); + }; + + render() { + const { config, authProviderOptions, regions } = this.state; + + return ( + <> +

CloudWatch Details

+
+
+
+ Auth Provider + ) => + this.onCredentialProfileNameChange(event.target.value) + } + /> +
+
+
+ )} + {config.jsonData.authType === 'keys' && ( +
+ {config.secureJsonFields.accessKey ? ( +
+
+ Access Key ID + +
+
+
+ +
+
+
+ ) : ( +
+
+ Access Key ID +
+ ) => this.onAccessKeyChange(event.target.value)} + /> +
+
+
+ )} + {config.secureJsonFields.secretKey ? ( +
+
+ Secret Access Key + +
+
+
+ +
+
+
+ ) : ( +
+
+ Secret Access Key +
+ ) => this.onSecretKeyChange(event.target.value)} + /> +
+
+
+ )} +
+ )} + {config.jsonData.authType === 'arn' && ( +
+
+ + Assume Role ARN + +
+ ) => this.onArnAssumeRoleChange(event.target.value)} + /> +
+
+
+ )} +
+
+ + Default Region + + ) => + this.onCustomMetricsNamespacesChange(event.target.value) + } + /> +
+
+ + + ); + } +} + +export default ConfigEditor; diff --git a/public/app/plugins/datasource/cloudwatch/components/Dimensions.test.tsx b/public/app/plugins/datasource/cloudwatch/components/Dimensions.test.tsx new file mode 100644 index 00000000000..43b03071d95 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Dimensions.test.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import { mount, shallow } from 'enzyme'; +import { Dimensions } from './'; +import { SelectableStrings } from '../types'; + +describe('Dimensions', () => { + it('renders', () => { + mount( + console.log(dimensions)} + loadKeys={() => Promise.resolve([])} + loadValues={() => Promise.resolve([])} + /> + ); + }); + + describe('and no dimension were passed to the component', () => { + it('initially displays just an add button', () => { + const wrapper = shallow( + {}} + loadKeys={() => Promise.resolve([])} + loadValues={() => Promise.resolve([])} + /> + ); + + expect(wrapper.html()).toEqual( + `
` + ); + }); + }); + + describe('and one dimension key along with a value were passed to the component', () => { + it('initially displays the dimension key, value and an add button', () => { + const wrapper = shallow( + {}} + loadKeys={() => Promise.resolve([])} + loadValues={() => Promise.resolve([])} + /> + ); + expect(wrapper.html()).toEqual( + `
somekey
somevalue
` + ); + }); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/Dimensions.tsx b/public/app/plugins/datasource/cloudwatch/components/Dimensions.tsx new file mode 100644 index 00000000000..e04d9ef29ac --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Dimensions.tsx @@ -0,0 +1,80 @@ +import React, { FunctionComponent, Fragment, useState, useEffect } from 'react'; +import isEqual from 'lodash/isEqual'; +import { SelectableValue } from '@grafana/data'; +import { SegmentAsync } from '@grafana/ui'; +import { SelectableStrings } from '../types'; + +export interface Props { + dimensions: { [key: string]: string | string[] }; + onChange: (dimensions: { [key: string]: string }) => void; + loadValues: (key: string) => Promise; + loadKeys: () => Promise; +} + +const removeText = '-- remove dimension --'; +const removeOption: SelectableValue = { label: removeText, value: removeText }; + +// The idea of this component is that is should only trigger the onChange event in the case +// there is a complete dimension object. E.g, when a new key is added is doesn't have a value. +// That should not trigger onChange. +export const Dimensions: FunctionComponent = ({ dimensions, loadValues, loadKeys, onChange }) => { + const [data, setData] = useState(dimensions); + + useEffect(() => { + const completeDimensions = Object.entries(data).reduce( + (res, [key, value]) => (value ? { ...res, [key]: value } : res), + {} + ); + if (!isEqual(completeDimensions, dimensions)) { + onChange(completeDimensions); + } + }, [data]); + + const excludeUsedKeys = (options: SelectableStrings) => { + return options.filter(({ value }) => !Object.keys(data).includes(value)); + }; + + return ( + <> + {Object.entries(data).map(([key, value], index) => ( + + loadKeys().then(keys => [removeOption, ...excludeUsedKeys(keys)])} + onChange={newKey => { + const { [key]: value, ...newDimensions } = data; + if (newKey === removeText) { + setData({ ...newDimensions }); + } else { + setData({ ...newDimensions, [newKey]: '' }); + } + }} + /> + + loadValues(key)} + onChange={newValue => setData({ ...data, [key]: newValue })} + /> + {Object.values(data).length > 1 && index + 1 !== Object.values(data).length && ( + + )} + + ))} + {Object.values(data).every(v => v) && ( + + + + } + loadOptions={() => loadKeys().then(excludeUsedKeys)} + onChange={(newKey: string) => setData({ ...data, [newKey]: '' })} + /> + )} + + ); +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/Forms.tsx b/public/app/plugins/datasource/cloudwatch/components/Forms.tsx new file mode 100644 index 00000000000..1e9cb61d5f7 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Forms.tsx @@ -0,0 +1,28 @@ +import React, { InputHTMLAttributes, FunctionComponent } from 'react'; +import { FormLabel } from '@grafana/ui'; + +export interface Props extends InputHTMLAttributes { + label: string; + tooltip?: string; + children?: React.ReactNode; +} + +export const QueryField: FunctionComponent> = ({ label, tooltip, children }) => ( + <> + + {label} + + {children} + +); + +export const QueryInlineField: FunctionComponent = ({ ...props }) => { + return ( +
+ +
+
+
+
+ ); +}; diff --git a/public/app/plugins/datasource/cloudwatch/components/QueryEditor.test.tsx b/public/app/plugins/datasource/cloudwatch/components/QueryEditor.test.tsx new file mode 100644 index 00000000000..bf8c77bbff5 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/QueryEditor.test.tsx @@ -0,0 +1,102 @@ +import React from 'react'; +import renderer from 'react-test-renderer'; +import { mount } from 'enzyme'; +import { DataSourceInstanceSettings } from '@grafana/data'; +import { TemplateSrv } from 'app/features/templating/template_srv'; +import { CustomVariable } from 'app/features/templating/all'; +import { QueryEditor, Props } from './QueryEditor'; +import CloudWatchDatasource from '../datasource'; + +const setup = () => { + const instanceSettings = { + jsonData: { defaultRegion: 'us-east-1' }, + } as DataSourceInstanceSettings; + + const templateSrv = new TemplateSrv(); + templateSrv.init([ + new CustomVariable( + { + name: 'var3', + options: [ + { selected: true, value: 'var3-foo' }, + { selected: false, value: 'var3-bar' }, + { selected: true, value: 'var3-baz' }, + ], + current: { + value: ['var3-foo', 'var3-baz'], + }, + multi: true, + }, + {} as any + ), + ]); + + const datasource = new CloudWatchDatasource(instanceSettings, {} as any, {} as any, templateSrv as any, {} as any); + datasource.metricFindQuery = async param => [{ value: 'test', label: 'test' }]; + + const props: Props = { + query: { + refId: '', + id: '', + region: 'us-east-1', + namespace: 'ec2', + metricName: 'CPUUtilization', + dimensions: { somekey: 'somevalue' }, + statistics: new Array(), + period: '', + expression: '', + alias: '', + highResolution: false, + matchExact: true, + }, + datasource, + onChange: jest.fn(), + onRunQuery: jest.fn(), + }; + + return props; +}; + +describe('QueryEditor', () => { + it('should render component', () => { + const props = setup(); + const tree = renderer.create().toJSON(); + expect(tree).toMatchSnapshot(); + }); + + describe('should use correct default values', () => { + it('when region is null is display default in the label', () => { + const props = setup(); + props.query.region = null; + const wrapper = mount(); + expect( + wrapper + .find('.gf-form-inline') + .first() + .find('.gf-form-label.query-part') + .first() + .text() + ).toEqual('default'); + }); + + it('should init props correctly', () => { + const props = setup(); + props.query.namespace = null; + props.query.metricName = null; + props.query.expression = null; + props.query.dimensions = null; + props.query.region = null; + props.query.statistics = null; + const wrapper = mount(); + const { + query: { namespace, region, metricName, dimensions, statistics, expression }, + } = wrapper.props(); + expect(namespace).toEqual(''); + expect(metricName).toEqual(''); + expect(expression).toEqual(''); + expect(region).toEqual('default'); + expect(statistics).toEqual(['Average']); + expect(dimensions).toEqual({}); + }); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/QueryEditor.tsx b/public/app/plugins/datasource/cloudwatch/components/QueryEditor.tsx new file mode 100644 index 00000000000..9925a962e08 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/QueryEditor.tsx @@ -0,0 +1,277 @@ +import React, { PureComponent, ChangeEvent } from 'react'; +import { SelectableValue, QueryEditorProps } from '@grafana/data'; +import { Input, Segment, SegmentAsync, ValidationEvents, EventsWithValidation, Switch } from '@grafana/ui'; +import { CloudWatchQuery } from '../types'; +import CloudWatchDatasource from '../datasource'; +import { SelectableStrings } from '../types'; +import { Stats, Dimensions, QueryInlineField, QueryField, Alias } from './'; + +export type Props = QueryEditorProps; + +interface State { + regions: SelectableStrings; + namespaces: SelectableStrings; + metricNames: SelectableStrings; + variableOptionGroup: SelectableValue; + showMeta: boolean; +} + +const idValidationEvents: ValidationEvents = { + [EventsWithValidation.onBlur]: [ + { + rule: value => new RegExp(/^$|^[a-z][a-zA-Z0-9_]*$/).test(value), + errorMessage: 'Invalid format. Only alphanumeric characters and underscores are allowed', + }, + ], +}; + +export class QueryEditor extends PureComponent { + state: State = { regions: [], namespaces: [], metricNames: [], variableOptionGroup: {}, showMeta: false }; + + componentWillMount() { + const { query } = this.props; + + if (!query.namespace) { + query.namespace = ''; + } + + if (!query.metricName) { + query.metricName = ''; + } + + if (!query.expression) { + query.expression = ''; + } + + if (!query.dimensions) { + query.dimensions = {}; + } + + if (!query.region) { + query.region = 'default'; + } + + if (!query.statistics || !query.statistics.length) { + query.statistics = ['Average']; + } + + if (!query.hasOwnProperty('highResolution')) { + query.highResolution = false; + } + + if (!query.hasOwnProperty('matchExact')) { + query.matchExact = true; + } + } + + componentDidMount() { + const { datasource } = this.props; + const variableOptionGroup = { + label: 'Template Variables', + options: this.props.datasource.variables.map(this.toOption), + }; + Promise.all([datasource.metricFindQuery('regions()'), datasource.metricFindQuery('namespaces()')]).then( + ([regions, namespaces]) => { + this.setState({ + ...this.state, + regions: [...regions, variableOptionGroup], + namespaces: [...namespaces, variableOptionGroup], + variableOptionGroup, + }); + } + ); + } + + loadMetricNames = async () => { + const { namespace, region } = this.props.query; + return this.props.datasource.metricFindQuery(`metrics(${namespace},${region})`).then(this.appendTemplateVariables); + }; + + appendTemplateVariables = (values: SelectableValue[]) => [ + ...values, + { label: 'Template Variables', options: this.props.datasource.variables.map(this.toOption) }, + ]; + + toOption = (value: any) => ({ label: value, value }); + + onChange(query: CloudWatchQuery) { + const { onChange, onRunQuery } = this.props; + onChange(query); + onRunQuery(); + } + + render() { + const { query, datasource, onChange, onRunQuery, data } = this.props; + const { regions, namespaces, variableOptionGroup: variableOptionGroup, showMeta } = this.state; + const metaDataExist = data && Object.values(data).length && data.state === 'Done'; + return ( + <> + + this.onChange({ ...query, region })} + /> + + + {query.expression.length === 0 && ( + <> + + this.onChange({ ...query, namespace })} + /> + + + + this.onChange({ ...query, metricName })} + /> + + + + this.onChange({ ...query, statistics })} + variableOptionGroup={variableOptionGroup} + /> + + + + this.onChange({ ...query, dimensions })} + loadKeys={() => + datasource.getDimensionKeys(query.namespace, query.region).then(this.appendTemplateVariables) + } + loadValues={newKey => { + const { [newKey]: value, ...newDimensions } = query.dimensions; + return datasource + .getDimensionValues(query.region, query.namespace, query.metricName, newKey, newDimensions) + .then(this.appendTemplateVariables); + }} + /> + + + )} + {query.statistics.length <= 1 && ( +
+
+ + ) => onChange({ ...query, id: event.target.value })} + validationEvents={idValidationEvents} + value={query.id || ''} + /> + +
+
+ + ) => + onChange({ ...query, expression: event.target.value }) + } + /> + +
+
+ )} +
+
+ + ) => onChange({ ...query, period: event.target.value })} + /> + +
+
+ + this.onChange({ ...query, alias: value })} /> + + this.onChange({ ...query, highResolution: !query.highResolution })} + /> + this.onChange({ ...query, matchExact: !query.matchExact })} + /> + +
+
+
+
+ {showMeta && metaDataExist && ( + + + + + + + + + {data.series[0].meta.gmdMeta.map(({ ID, Expression }: any) => ( + + + + + ))} + +
Metric Data Query IDMetric Data Query Expression +
{ID}{Expression}
+ )} +
+ + ); + } +} diff --git a/public/app/plugins/datasource/cloudwatch/components/Stats.test.tsx b/public/app/plugins/datasource/cloudwatch/components/Stats.test.tsx new file mode 100644 index 00000000000..a4d3b8f146b --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Stats.test.tsx @@ -0,0 +1,21 @@ +import React from 'react'; +import renderer from 'react-test-renderer'; +import { Stats } from './Stats'; + +const toOption = (value: any) => ({ label: value, value }); + +describe('Stats', () => { + it('should render component', () => { + const tree = renderer + .create( + {}} + stats={['Average', 'Maximum', 'Minimum', 'Sum', 'SampleCount'].map(toOption)} + /> + ) + .toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/public/app/plugins/datasource/cloudwatch/components/Stats.tsx b/public/app/plugins/datasource/cloudwatch/components/Stats.tsx new file mode 100644 index 00000000000..e33f333eb89 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/Stats.tsx @@ -0,0 +1,47 @@ +import React, { FunctionComponent } from 'react'; +import { SelectableStrings } from '../types'; +import { SelectableValue } from '@grafana/data'; +import { Segment } from '@grafana/ui'; + +export interface Props { + values: string[]; + onChange: (values: string[]) => void; + variableOptionGroup: SelectableValue; + stats: SelectableStrings; +} + +const removeText = '-- remove stat --'; +const removeOption: SelectableValue = { label: removeText, value: removeText }; + +export const Stats: FunctionComponent = ({ stats, values, onChange, variableOptionGroup }) => ( + <> + {values && + values.map((value, index) => ( + + onChange( + value === removeText + ? values.filter((_, i) => i !== index) + : values.map((v, i) => (i === index ? value : v)) + ) + } + /> + ))} + {values.length !== stats.length && ( + + + + } + allowCustomValue + onChange={(value: string) => onChange([...values, value])} + options={[...stats.filter(({ value }) => !values.includes(value)), variableOptionGroup]} + /> + )} + +); diff --git a/public/app/plugins/datasource/cloudwatch/components/ThrottlingErrorMessage.tsx b/public/app/plugins/datasource/cloudwatch/components/ThrottlingErrorMessage.tsx new file mode 100644 index 00000000000..11fb3def37a --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/ThrottlingErrorMessage.tsx @@ -0,0 +1,27 @@ +import React, { FunctionComponent } from 'react'; + +export interface Props { + region: string; +} + +export const ThrottlingErrorMessage: FunctionComponent = ({ region }) => ( +

+ Please visit the  + + AWS Service Quotas console + +  to request a quota increase or see our  + + documentation + +  to learn more. +

+); diff --git a/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Alias.test.tsx.snap b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Alias.test.tsx.snap new file mode 100644 index 00000000000..0b5ae814517 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Alias.test.tsx.snap @@ -0,0 +1,18 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Alias should render component 1`] = ` +
+ +
+`; diff --git a/public/app/plugins/datasource/cloudwatch/components/__snapshots__/ConfigEditor.test.tsx.snap b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/ConfigEditor.test.tsx.snap new file mode 100644 index 00000000000..6f27bc6bd76 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/ConfigEditor.test.tsx.snap @@ -0,0 +1,901 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Render should disable access key id field 1`] = ` + +

+ CloudWatch Details +

+
+
+
+ + Auth Provider + + +
+
+
+
+
+ + Secret Access Key + +
+ +
+
+
+
+
+
+ + Default Region + + +
+
+
+ +`; + +exports[`Render should render component 1`] = ` + +

+ CloudWatch Details +

+
+
+
+ + Auth Provider + + +
+
+
+
+
+ + Secret Access Key + +
+ +
+
+
+ +
+
+ + Default Region + + +
+
+ + +`; + +exports[`Render should should show access key and secret access key fields 1`] = ` + +

+ CloudWatch Details +

+
+
+
+ + Auth Provider + + +
+
+
+
+
+ + Secret Access Key + +
+ +
+
+
+ +
+
+ + Default Region + + +
+
+ + +`; + +exports[`Render should should show arn role field 1`] = ` + +

+ CloudWatch Details +

+
+
+
+ + Auth Provider + + +
+
+
+
+
+ + Secret Access Key + +
+ +
+
+
+ +
+
+ + Default Region + + +
+
+ + +`; + +exports[`Render should should show credentials profile name field 1`] = ` + +

+ CloudWatch Details +

+
+
+
+ + Auth Provider + + +
+
+
+
+
+ + Secret Access Key + +
+ +
+
+
+ +
+
+ + Default Region + + +
+
+ + +`; diff --git a/public/app/plugins/datasource/cloudwatch/components/__snapshots__/QueryEditor.test.tsx.snap b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/QueryEditor.test.tsx.snap new file mode 100644 index 00000000000..1851288770a --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/QueryEditor.test.tsx.snap @@ -0,0 +1,396 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`QueryEditor should render component 1`] = ` +Array [ +
+ + +
+
+
+
, +
+ + +
+
+
+
, +
+ + +
+
+
+
, +
+ + +
+ + + +
+
+
+
+
, +
+ + + + +
+ + + +
+
+
+
+
, +
+
+ +
+ +
+
+
+ +
+ +
+
+
, +
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+
+ +
+ +
+
+
+
+
, +] +`; diff --git a/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Stats.test.tsx.snap b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Stats.test.tsx.snap new file mode 100644 index 00000000000..e4bae094d06 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/__snapshots__/Stats.test.tsx.snap @@ -0,0 +1,38 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Stats should render component 1`] = ` +Array [ + , + , +
+ + + +
, +] +`; diff --git a/public/app/plugins/datasource/cloudwatch/components/index.ts b/public/app/plugins/datasource/cloudwatch/components/index.ts new file mode 100644 index 00000000000..8bb770ce20d --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/components/index.ts @@ -0,0 +1,4 @@ +export { Stats } from './Stats'; +export { Dimensions } from './Dimensions'; +export { QueryInlineField, QueryField } from './Forms'; +export { Alias } from './Alias'; diff --git a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts b/public/app/plugins/datasource/cloudwatch/config_ctrl.ts deleted file mode 100644 index cb172dcc673..00000000000 --- a/public/app/plugins/datasource/cloudwatch/config_ctrl.ts +++ /dev/null @@ -1,89 +0,0 @@ -import _ from 'lodash'; -import DatasourceSrv from 'app/features/plugins/datasource_srv'; -import CloudWatchDatasource from './datasource'; -export class CloudWatchConfigCtrl { - static templateUrl = 'partials/config.html'; - current: any; - datasourceSrv: any; - - accessKeyExist = false; - secretKeyExist = false; - - /** @ngInject */ - constructor($scope: any, datasourceSrv: DatasourceSrv) { - this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp'; - this.current.jsonData.authType = this.current.jsonData.authType || 'credentials'; - - this.accessKeyExist = this.current.secureJsonFields.accessKey; - this.secretKeyExist = this.current.secureJsonFields.secretKey; - this.datasourceSrv = datasourceSrv; - this.getRegions(); - } - - resetAccessKey() { - this.accessKeyExist = false; - } - - resetSecretKey() { - this.secretKeyExist = false; - } - - authTypes = [ - { name: 'Access & secret key', value: 'keys' }, - { name: 'Credentials file', value: 'credentials' }, - { name: 'ARN', value: 'arn' }, - ]; - - indexPatternTypes: any = [ - { name: 'No pattern', value: undefined }, - { name: 'Hourly', value: 'Hourly', example: '[logstash-]YYYY.MM.DD.HH' }, - { name: 'Daily', value: 'Daily', example: '[logstash-]YYYY.MM.DD' }, - { name: 'Weekly', value: 'Weekly', example: '[logstash-]GGGG.WW' }, - { name: 'Monthly', value: 'Monthly', example: '[logstash-]YYYY.MM' }, - { name: 'Yearly', value: 'Yearly', example: '[logstash-]YYYY' }, - ]; - - regions = [ - 'ap-east-1', - 'ap-northeast-1', - 'ap-northeast-2', - 'ap-northeast-3', - 'ap-south-1', - 'ap-southeast-1', - 'ap-southeast-2', - 'ca-central-1', - 'cn-north-1', - 'cn-northwest-1', - 'eu-central-1', - 'eu-north-1', - 'eu-west-1', - 'eu-west-2', - 'eu-west-3', - 'me-south-1', - 'sa-east-1', - 'us-east-1', - 'us-east-2', - 'us-gov-east-1', - 'us-gov-west-1', - 'us-iso-east-1', - 'us-isob-east-1', - 'us-west-1', - 'us-west-2', - ]; - - getRegions() { - this.datasourceSrv - .loadDatasource(this.current.name) - .then((ds: CloudWatchDatasource) => { - return ds.getRegions(); - }) - .then( - (regions: any) => { - this.regions = _.map(regions, 'value'); - }, - (err: any) => { - console.error('failed to get latest regions'); - } - ); - } -} diff --git a/public/app/plugins/datasource/cloudwatch/dashboards/EBS.json b/public/app/plugins/datasource/cloudwatch/dashboards/EBS.json new file mode 100644 index 00000000000..f1c4e48f20e --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/dashboards/EBS.json @@ -0,0 +1,1024 @@ +{ + "__inputs": [ + { + "name": "DS_CLOUDWATCH", + "label": "CloudWatch", + "description": "", + "type": "datasource", + "pluginId": "cloudwatch", + "pluginName": "CloudWatch" + } + ], + "__requires": [ + { + "type": "datasource", + "id": "cloudwatch", + "name": "CloudWatch", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.6.0-pre" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1573630293215, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeReadBytes", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Read Bytes $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeWriteBytes", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Write Bytes $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeTotalReadTime", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Total Read Time $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeTotalWriteTime", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Total Write Time $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeReadOps", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Read Ops $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeWriteOps", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Write Ops $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeIdleTime", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Idle Time $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "VolumeQueueLength", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Volume Queue Length $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "VolumeId": "$volume" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "BurstBalance", + "namespace": "AWS/EBS", + "refId": "A", + "region": "$region", + "statistics": ["Average"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Burst Balance Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "CloudWatch", + "value": "CloudWatch" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "cloudwatch", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "default", + "value": "default" + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "regions()", + "hide": 0, + "includeAll": false, + "label": "Region", + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_CLOUDWATCH}", + "definition": "statistics()", + "hide": 0, + "includeAll": false, + "label": "Statistic", + "multi": false, + "name": "statistic", + "options": [], + "query": "statistics()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "*", + "value": ["*"] + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "dimension_values($region, AWS/EBS, , VolumeId)", + "hide": 0, + "includeAll": true, + "label": "Volume", + "multi": true, + "name": "volume", + "options": [], + "query": "dimension_values($region, AWS/EBS, , VolumeId)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] + }, + "timezone": "", + "title": "EBS", + "uid": "VgpJGb1Za", + "version": 10 +} diff --git a/public/app/plugins/datasource/cloudwatch/dashboards/Lambda.json b/public/app/plugins/datasource/cloudwatch/dashboards/Lambda.json new file mode 100644 index 00000000000..e6d5190488e --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/dashboards/Lambda.json @@ -0,0 +1,545 @@ +{ + "__inputs": [ + { + "name": "DS_CLOUDWATCH", + "label": "CloudWatch", + "description": "", + "type": "datasource", + "pluginId": "cloudwatch", + "pluginName": "CloudWatch" + } + ], + "__requires": [ + { + "type": "datasource", + "id": "cloudwatch", + "name": "CloudWatch", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.6.0-pre" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1573631164529, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "FunctionName": "$function" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "Invocations", + "namespace": "AWS/Lambda", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Invocations $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "FunctionName": "$function" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "Duration", + "namespace": "AWS/Lambda", + "refId": "A", + "region": "$region", + "statistics": ["Average"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Duration Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "FunctionName": "$function" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "Errors", + "namespace": "AWS/Lambda", + "refId": "A", + "region": "$region", + "statistics": ["Average"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "FunctionName": "$function" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "Throttles", + "namespace": "AWS/Lambda", + "refId": "A", + "region": "$region", + "statistics": ["Average"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttles $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "CloudWatch", + "value": "CloudWatch" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "cloudwatch", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "default", + "value": "default" + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "regions()", + "hide": 0, + "includeAll": false, + "label": "Region", + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_CLOUDWATCH}", + "definition": "statistics()", + "hide": 0, + "includeAll": false, + "label": "Statistic", + "multi": false, + "name": "statistic", + "options": [], + "query": "statistics()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "*", + "value": ["*"] + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "dimension_values($region, AWS/Lambda, , FunctionName)", + "hide": 0, + "includeAll": true, + "label": "FunctionName", + "multi": true, + "name": "function", + "options": [], + "query": "dimension_values($region, AWS/Lambda, , FunctionName)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] + }, + "timezone": "", + "title": "Lambda", + "uid": "VgpJGb1Zg", + "version": 6 +} diff --git a/public/app/plugins/datasource/cloudwatch/dashboards/ec2.json b/public/app/plugins/datasource/cloudwatch/dashboards/ec2.json new file mode 100644 index 00000000000..5f046323eee --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/dashboards/ec2.json @@ -0,0 +1,1220 @@ +{ + "__inputs": [ + { + "name": "DS_CLOUDWATCH", + "label": "CloudWatch", + "description": "", + "type": "datasource", + "pluginId": "cloudwatch", + "pluginName": "CloudWatch" + } + ], + "__requires": [ + { + "type": "datasource", + "id": "cloudwatch", + "name": "CloudWatch", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.6.0-pre" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1573628451316, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "CPUUtilization", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "DiskReadBytes", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Read Bytes $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "DiskReadOps", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Read Ops $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "DiskWriteOps", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Write Bytes $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "DiskWriteOps", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk Write Ops $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "Network In", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network In $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "NetworkOut", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Out $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "NetworkPacketsIn", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["$statistic"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Packets In $statistic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "StatusCheckFailed", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["Sum"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Status Check Failed Sum", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 36 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "StatusCheckFailed_Instance", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["Sum"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Status Check Failed Instance Sum", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_CLOUDWATCH}", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": { + "InstanceId": "$instance" + }, + "expression": "", + "highResolution": false, + "matchExact": true, + "metricName": "StatusCheckFailed_System", + "namespace": "AWS/EC2", + "refId": "A", + "region": "$region", + "statistics": ["Sum"] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Status Check Failed System Sum", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "tags": [], + "text": "CloudWatch", + "value": "CloudWatch" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "cloudwatch", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "default", + "value": "default" + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "regions()", + "hide": 0, + "includeAll": false, + "label": "Region", + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_CLOUDWATCH}", + "definition": "statistics()", + "hide": 0, + "includeAll": false, + "label": "Statistic", + "multi": false, + "name": "statistic", + "options": [], + "query": "statistics()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "*", + "value": ["*"] + }, + "datasource": "${DS_CLOUDWATCH}", + "definition": "dimension_values($region, AWS/EC2, , InstanceId)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "dimension_values($region, AWS/EC2, , InstanceId)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] + }, + "timezone": "", + "title": "EC2", + "uid": "VgpJGb1Zk", + "version": 1 +} diff --git a/public/app/plugins/datasource/cloudwatch/datasource.ts b/public/app/plugins/datasource/cloudwatch/datasource.ts index dd7428e20e5..d1ca1daf9a4 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.ts @@ -1,5 +1,11 @@ +import React from 'react'; import angular, { IQService } from 'angular'; import _ from 'lodash'; +import { notifyApp } from 'app/core/actions'; +import { createErrorNotification } from 'app/core/copy/appNotification'; +import { AppNotificationTimeout } from 'app/types'; +import { store } from 'app/store/store'; +import kbn from 'app/core/utils/kbn'; import { dateMath, ScopedVars, @@ -9,22 +15,39 @@ import { DataQueryRequest, DataSourceInstanceSettings, } from '@grafana/data'; -import kbn from 'app/core/utils/kbn'; -import { CloudWatchQuery } from './types'; import { BackendSrv } from 'app/core/services/backend_srv'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { TimeSrv } from 'app/features/dashboard/services/TimeSrv'; -// import * as moment from 'moment'; - -export default class CloudWatchDatasource extends DataSourceApi { +import { ThrottlingErrorMessage } from './components/ThrottlingErrorMessage'; +import memoizedDebounce from './memoizedDebounce'; +import { CloudWatchQuery, CloudWatchJsonData } from './types'; + +const displayAlert = (datasourceName: string, region: string) => + store.dispatch( + notifyApp( + createErrorNotification( + `CloudWatch request limit reached in ${region} for data source ${datasourceName}`, + '', + React.createElement(ThrottlingErrorMessage, { region }, null) + ) + ) + ); + +const displayCustomError = (title: string, message: string) => + store.dispatch(notifyApp(createErrorNotification(title, message))); + +export default class CloudWatchDatasource extends DataSourceApi { type: any; proxyUrl: any; defaultRegion: any; standardStatistics: any; + datasourceName: string; + debouncedAlert: (datasourceName: string, region: string) => void; + debouncedCustomAlert: (title: string, message: string) => void; /** @ngInject */ constructor( - private instanceSettings: DataSourceInstanceSettings, + instanceSettings: DataSourceInstanceSettings, private $q: IQService, private backendSrv: BackendSrv, private templateSrv: TemplateSrv, @@ -34,13 +57,14 @@ export default class CloudWatchDatasource extends DataSourceApi this.type = 'cloudwatch'; this.proxyUrl = instanceSettings.url; this.defaultRegion = instanceSettings.jsonData.defaultRegion; - this.instanceSettings = instanceSettings; + this.datasourceName = instanceSettings.name; this.standardStatistics = ['Average', 'Maximum', 'Minimum', 'Sum', 'SampleCount']; + this.debouncedAlert = memoizedDebounce(displayAlert, AppNotificationTimeout.Error); + this.debouncedCustomAlert = memoizedDebounce(displayCustomError, AppNotificationTimeout.Error); } query(options: DataQueryRequest) { options = angular.copy(options); - options.targets = this.expandTemplateVariable(options.targets, options.scopedVars, this.templateSrv); const queries = _.filter(options.targets, item => { return ( @@ -49,16 +73,14 @@ export default class CloudWatchDatasource extends DataSourceApi item.expression.length > 0) ); }).map(item => { - item.region = this.templateSrv.replace(this.getActualRegion(item.region), options.scopedVars); - item.namespace = this.templateSrv.replace(item.namespace, options.scopedVars); - item.metricName = this.templateSrv.replace(item.metricName, options.scopedVars); + item.region = this.replace(this.getActualRegion(item.region), options.scopedVars, true, 'region'); + item.namespace = this.replace(item.namespace, options.scopedVars, true, 'namespace'); + item.metricName = this.replace(item.metricName, options.scopedVars, true, 'metric name'); item.dimensions = this.convertDimensionFormat(item.dimensions, options.scopedVars); - item.statistics = item.statistics.map(s => { - return this.templateSrv.replace(s, options.scopedVars); - }); + item.statistics = item.statistics.map(stat => this.replace(stat, options.scopedVars, true, 'statistics')); item.period = String(this.getPeriod(item, options)); // use string format for period in graph query, and alerting - item.id = this.templateSrv.replace(item.id, options.scopedVars); - item.expression = this.templateSrv.replace(item.expression, options.scopedVars); + item.id = this.replace(item.id, options.scopedVars, true, 'id'); + item.expression = this.replace(item.expression, options.scopedVars, true, 'expression'); // valid ExtendedStatistics is like p90.00, check the pattern const hasInvalidStatistics = item.statistics.some(s => { @@ -79,7 +101,7 @@ export default class CloudWatchDatasource extends DataSourceApi refId: item.refId, intervalMs: options.intervalMs, maxDataPoints: options.maxDataPoints, - datasourceId: this.instanceSettings.id, + datasourceId: this.id, type: 'timeSeriesQuery', }, item @@ -102,6 +124,10 @@ export default class CloudWatchDatasource extends DataSourceApi return this.performTimeSeriesQuery(request, options.range); } + get variables() { + return this.templateSrv.variables.map(v => `$${v.name}`); + } + getPeriod(target: any, options: any, now?: number) { const start = this.convertToCloudWatchTime(options.range.from, false); const end = this.convertToCloudWatchTime(options.range.to, true); @@ -149,30 +175,50 @@ export default class CloudWatchDatasource extends DataSourceApi } buildCloudwatchConsoleUrl( - { region, namespace, metricName, dimensions, statistics, period }: CloudWatchQuery, + { region, namespace, metricName, dimensions, statistics, period, expression }: CloudWatchQuery, start: string, end: string, - title: string + title: string, + gmdMeta: Array<{ Expression: string }> ) { - const conf = { + region = this.getActualRegion(region); + let conf = { view: 'timeSeries', stacked: false, title, start, end, region, - metrics: [ - ...statistics.map(stat => [ - namespace, - metricName, - ...Object.entries(dimensions).reduce((acc, [key, value]) => [...acc, key, value], []), - { - stat, - period, - }, - ]), - ], - }; + } as any; + + const isSearchExpression = + gmdMeta && gmdMeta.length && gmdMeta.every(({ Expression: expression }) => /SEARCH().*/.test(expression)); + const isMathExpression = !isSearchExpression && expression; + + if (isMathExpression) { + return ''; + } + + if (isSearchExpression) { + const metrics: any = + gmdMeta && gmdMeta.length ? gmdMeta.map(({ Expression: expression }) => ({ expression })) : [{ expression }]; + conf = { ...conf, metrics }; + } else { + conf = { + ...conf, + metrics: [ + ...statistics.map(stat => [ + namespace, + metricName, + ...Object.entries(dimensions).reduce((acc, [key, value]) => [...acc, key, value[0]], []), + { + stat, + period, + }, + ]), + ], + }; + } return `https://${region}.console.aws.amazon.com/cloudwatch/deeplink.js?region=${region}#metricsV2:graph=${encodeURIComponent( JSON.stringify(conf) @@ -180,44 +226,70 @@ export default class CloudWatchDatasource extends DataSourceApi } performTimeSeriesQuery(request: any, { from, to }: TimeRange) { - return this.awsRequest('/api/tsdb/query', request).then((res: any) => { - if (!res.results) { - return { data: [] }; - } - const dataFrames = Object.values(request.queries).reduce((acc: any, queryRequest: any) => { - const queryResult = res.results[queryRequest.refId]; - if (!queryResult) { - return acc; + return this.awsRequest('/api/tsdb/query', request) + .then((res: any) => { + if (!res.results) { + return { data: [] }; } + return Object.values(request.queries).reduce( + ({ data, error }: any, queryRequest: any) => { + const queryResult = res.results[queryRequest.refId]; + if (!queryResult) { + return { data, error }; + } - const link = this.buildCloudwatchConsoleUrl( - queryRequest, - from.toISOString(), - to.toISOString(), - `query${queryRequest.refId}` + const link = this.buildCloudwatchConsoleUrl( + queryRequest, + from.toISOString(), + to.toISOString(), + queryRequest.refId, + queryResult.meta.gmdMeta + ); + + return { + error: error || queryResult.error ? { message: queryResult.error } : null, + data: [ + ...data, + ...queryResult.series.map(({ name, points }: any) => { + const dataFrame = toDataFrame({ + target: name, + datapoints: points, + refId: queryRequest.refId, + meta: queryResult.meta, + }); + if (link) { + for (const field of dataFrame.fields) { + field.config.links = [ + { + url: link, + title: 'View in CloudWatch console', + targetBlank: true, + }, + ]; + } + } + return dataFrame; + }), + ], + }; + }, + { data: [], error: null } ); + }) + .catch((err: any = { data: { error: '' } }) => { + if (/^Throttling:.*/.test(err.data.message)) { + const failedRedIds = Object.keys(err.data.results); + const regionsAffected = Object.values(request.queries).reduce( + (res: string[], { refId, region }: CloudWatchQuery) => + !failedRedIds.includes(refId) || res.includes(region) ? res : [...res, region], + [] + ) as string[]; + + regionsAffected.forEach(region => this.debouncedAlert(this.datasourceName, this.getActualRegion(region))); + } - return [ - ...acc, - ...queryResult.series.map(({ name, points, meta }: any) => { - const series = { target: name, datapoints: points }; - const dataFrame = toDataFrame(meta && meta.unit ? { ...series, unit: meta.unit } : series); - for (const field of dataFrame.fields) { - field.config.links = [ - { - url: link, - title: 'View in CloudWatch console', - targetBlank: true, - }, - ]; - } - return dataFrame; - }), - ]; - }, []); - - return { data: dataFrames }; - }); + throw err; + }); } transformSuggestDataFromTable(suggestData: any) { @@ -225,6 +297,7 @@ export default class CloudWatchDatasource extends DataSourceApi return { text: v[0], value: v[1], + label: v[1], }; }); } @@ -240,7 +313,7 @@ export default class CloudWatchDatasource extends DataSourceApi refId: 'metricFindQuery', intervalMs: 1, // dummy maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, + datasourceId: this.id, type: 'metricFindQuery', subtype: subtype, }, @@ -260,34 +333,48 @@ export default class CloudWatchDatasource extends DataSourceApi return this.doMetricQueryRequest('namespaces', null); } - getMetrics(namespace: string, region: string) { + async getMetrics(namespace: string, region: string) { + if (!namespace || !region) { + return []; + } + return this.doMetricQueryRequest('metrics', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), }); } - getDimensionKeys(namespace: string, region: string) { + async getDimensionKeys(namespace: string, region: string) { + if (!namespace) { + return []; + } + return this.doMetricQueryRequest('dimension_keys', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), }); } - getDimensionValues( + async getDimensionValues( region: string, namespace: string, metricName: string, dimensionKey: string, filterDimensions: {} ) { - return this.doMetricQueryRequest('dimension_values', { + if (!namespace || !metricName) { + return []; + } + + const values = await this.doMetricQueryRequest('dimension_values', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), - metricName: this.templateSrv.replace(metricName), + metricName: this.templateSrv.replace(metricName.trim()), dimensionKey: this.templateSrv.replace(dimensionKey), dimensions: this.convertDimensionFormat(filterDimensions, {}), }); + + return values.length ? [{ value: '*', text: '*', label: '*' }, ...values] : values; } getEbsVolumeIds(region: string, instanceId: string) { @@ -313,7 +400,7 @@ export default class CloudWatchDatasource extends DataSourceApi }); } - metricFindQuery(query: string) { + async metricFindQuery(query: string) { let region; let namespace; let metricName; @@ -382,6 +469,11 @@ export default class CloudWatchDatasource extends DataSourceApi return this.getResourceARNs(region, resourceType, tagsJSON); } + const statsQuery = query.match(/^statistics\(\)/); + if (statsQuery) { + return this.standardStatistics.map((s: string) => ({ value: s, label: s, text: s })); + } + return this.$q.when([]); } @@ -414,7 +506,7 @@ export default class CloudWatchDatasource extends DataSourceApi refId: 'annotationQuery', intervalMs: 1, // dummy maxDataPoints: 1, // dummy - datasourceId: this.instanceSettings.id, + datasourceId: this.id, type: 'annotationQuery', }, parameters @@ -445,7 +537,7 @@ export default class CloudWatchDatasource extends DataSourceApi } testDatasource() { - /* use billing metrics for test */ + // use billing metrics for test const region = this.defaultRegion; const namespace = 'AWS/Billing'; const metricName = 'EstimatedCharges'; @@ -479,80 +571,45 @@ export default class CloudWatchDatasource extends DataSourceApi return region; } - getExpandedVariables(target: any, dimensionKey: any, variable: any, templateSrv: TemplateSrv) { - /* if the all checkbox is marked we should add all values to the targets */ - const allSelected: any = _.find(variable.options, { selected: true, text: 'All' }); - const selectedVariables = _.filter(variable.options, v => { - if (allSelected) { - return v.text !== 'All'; - } else { - return v.selected; - } - }); - const currentVariables = !_.isArray(variable.current.value) - ? [variable.current] - : variable.current.value.map((v: any) => { - return { - text: v, - value: v, - }; - }); - const useSelectedVariables = - selectedVariables.some((s: any) => { - return s.value === currentVariables[0].value; - }) || currentVariables[0].value === '$__all'; - return (useSelectedVariables ? selectedVariables : currentVariables).map((v: any) => { - const t = angular.copy(target); - const scopedVar: any = {}; - scopedVar[variable.name] = v; - t.refId = target.refId + '_' + v.value; - t.dimensions[dimensionKey] = templateSrv.replace(t.dimensions[dimensionKey], scopedVar); - if (variable.multi && target.id) { - t.id = target.id + window.btoa(v.value).replace(/=/g, '0'); // generate unique id - } else { - t.id = target.id; - } - return t; - }); + convertToCloudWatchTime(date: any, roundUp: any) { + if (_.isString(date)) { + date = dateMath.parse(date, roundUp); + } + return Math.round(date.valueOf() / 1000); } - expandTemplateVariable(targets: any, scopedVars: ScopedVars, templateSrv: TemplateSrv) { - // Datasource and template srv logic uber-complected. This should be cleaned up. - return _.chain(targets) - .map(target => { - if (target.id && target.id.length > 0 && target.expression && target.expression.length > 0) { - return [target]; - } + convertDimensionFormat(dimensions: { [key: string]: string | string[] }, scopedVars: ScopedVars) { + return Object.entries(dimensions).reduce((result, [key, value]) => { + key = this.replace(key, scopedVars, true, 'dimension keys'); - const variableIndex = _.keyBy(templateSrv.variables, 'name'); - const dimensionKey = _.findKey(target.dimensions, v => { - const variableName = templateSrv.getVariableName(v); - return templateSrv.variableExists(v) && !_.has(scopedVars, variableName) && variableIndex[variableName].multi; - }); + if (Array.isArray(value)) { + return { ...result, [key]: value }; + } - if (dimensionKey) { - const multiVariable = variableIndex[templateSrv.getVariableName(target.dimensions[dimensionKey])]; - return this.getExpandedVariables(target, dimensionKey, multiVariable, templateSrv); - } else { - return [target]; + const valueVar = this.templateSrv.variables.find(({ name }) => name === this.templateSrv.getVariableName(value)); + if (valueVar) { + if (valueVar.multi) { + const values = this.templateSrv.replace(value, scopedVars, 'pipe').split('|'); + return { ...result, [key]: values }; } - }) - .flatten() - .value(); + return { ...result, [key]: [this.templateSrv.replace(value, scopedVars)] }; + } + + return { ...result, [key]: [value] }; + }, {}); } - convertToCloudWatchTime(date: any, roundUp: any) { - if (_.isString(date)) { - date = dateMath.parse(date, roundUp); + replace(target: string, scopedVars: ScopedVars, displayErrorIfIsMultiTemplateVariable?: boolean, fieldName?: string) { + if (displayErrorIfIsMultiTemplateVariable) { + const variable = this.templateSrv.variables.find(({ name }) => name === this.templateSrv.getVariableName(target)); + if (variable && variable.multi) { + this.debouncedCustomAlert( + 'CloudWatch templating error', + `Multi template variables are not supported for ${fieldName || target}` + ); + } } - return Math.round(date.valueOf() / 1000); - } - convertDimensionFormat(dimensions: any, scopedVars: ScopedVars) { - const convertedDimensions: any = {}; - _.each(dimensions, (value, key) => { - convertedDimensions[this.templateSrv.replace(key, scopedVars)] = this.templateSrv.replace(value, scopedVars); - }); - return convertedDimensions; + return this.templateSrv.replace(target, scopedVars); } } diff --git a/public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts b/public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts new file mode 100644 index 00000000000..2a661908b0a --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts @@ -0,0 +1,13 @@ +import { debounce, memoize } from 'lodash'; + +export default (func: (...args: any[]) => void, wait = 7000) => { + const mem = memoize( + (...args) => + debounce(func, wait, { + leading: true, + }), + (...args) => JSON.stringify(args) + ); + + return (...args: any[]) => mem(...args)(...args); +}; diff --git a/public/app/plugins/datasource/cloudwatch/module.ts b/public/app/plugins/datasource/cloudwatch/module.ts deleted file mode 100644 index 45decb8cb1f..00000000000 --- a/public/app/plugins/datasource/cloudwatch/module.ts +++ /dev/null @@ -1,16 +0,0 @@ -import './query_parameter_ctrl'; - -import CloudWatchDatasource from './datasource'; -import { CloudWatchQueryCtrl } from './query_ctrl'; -import { CloudWatchConfigCtrl } from './config_ctrl'; - -class CloudWatchAnnotationsQueryCtrl { - static templateUrl = 'partials/annotations.editor.html'; -} - -export { - CloudWatchDatasource as Datasource, - CloudWatchQueryCtrl as QueryCtrl, - CloudWatchConfigCtrl as ConfigCtrl, - CloudWatchAnnotationsQueryCtrl as AnnotationsQueryCtrl, -}; diff --git a/public/app/plugins/datasource/cloudwatch/module.tsx b/public/app/plugins/datasource/cloudwatch/module.tsx new file mode 100644 index 00000000000..8880acaaa64 --- /dev/null +++ b/public/app/plugins/datasource/cloudwatch/module.tsx @@ -0,0 +1,16 @@ +import { DataSourcePlugin } from '@grafana/data'; +import { ConfigEditor } from './components/ConfigEditor'; +import { QueryEditor } from './components/QueryEditor'; +import CloudWatchDatasource from './datasource'; +import { CloudWatchJsonData, CloudWatchQuery } from './types'; + +class CloudWatchAnnotationsQueryCtrl { + static templateUrl = 'partials/annotations.editor.html'; +} + +export const plugin = new DataSourcePlugin( + CloudWatchDatasource +) + .setConfigEditor(ConfigEditor) + .setQueryEditor(QueryEditor) + .setAnnotationQueryCtrl(CloudWatchAnnotationsQueryCtrl); diff --git a/public/app/plugins/datasource/cloudwatch/partials/config.html b/public/app/plugins/datasource/cloudwatch/partials/config.html deleted file mode 100644 index 0f74901ee19..00000000000 --- a/public/app/plugins/datasource/cloudwatch/partials/config.html +++ /dev/null @@ -1,55 +0,0 @@ -

CloudWatch details

- -
-
- - -
- -
- - - - Credentials profile name, as specified in ~/.aws/credentials, leave blank for default - -
- -
- - - Reset - -
- -
- - - Reset - -
- -
- - - - ARN of Assume Role - -
- -
- -
- - - Specify the region, such as for US West (Oregon) use ` us-west-2 ` as the region. - -
-
-
- - - - Namespaces of Custom Metrics - -
-
diff --git a/public/app/plugins/datasource/cloudwatch/partials/query.editor.html b/public/app/plugins/datasource/cloudwatch/partials/query.editor.html deleted file mode 100644 index 82b00e9a23b..00000000000 --- a/public/app/plugins/datasource/cloudwatch/partials/query.editor.html +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html b/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html index ce272bb4d64..1ccdae40f1a 100644 --- a/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html +++ b/public/app/plugins/datasource/cloudwatch/partials/query.parameter.html @@ -1,92 +1,143 @@
-
- - -
+
+ + +
-
-
-
+
+
+
-
- +
+ - - -
+ + +
-
- -
+
+ +
-
- -
+
+ +
-
-
-
+
+
+
-
- - -
+
+ + +
-
-
-
+
+
+
-
- - -
-
- - -
+
+ + +
+
+ + +
-
- - -
-
- - - - Alias replacement variables: -
    -
  • {{metric}}
    • -
  • {{stat}}
    • -
  • {{namespace}}
    • -
  • {{region}}
    • -
  • {{period}}
    • -
  • {{label}}
    • -
  • {{YOUR_DIMENSION_NAME}}
    • -
- -
-
- - -
+
+ + +
+
+ + + + Alias replacement variables: +
    +
  • {{ metric }}
    • +
  • {{ stat }}
    • +
  • {{ namespace }}
    • +
  • {{ region }}
    • +
  • {{ period }}
    • +
  • {{ label }}
    • +
  • {{ YOUR_DIMENSION_NAME }}
    • +
+ +
+
+ + +
-
-
-
+
+
+
diff --git a/public/app/plugins/datasource/cloudwatch/plugin.json b/public/app/plugins/datasource/cloudwatch/plugin.json index 212bb20a059..ee0ca7e088c 100644 --- a/public/app/plugins/datasource/cloudwatch/plugin.json +++ b/public/app/plugins/datasource/cloudwatch/plugin.json @@ -7,7 +7,6 @@ "metrics": true, "alerting": true, "annotations": true, - "info": { "description": "Data source for Amazon AWS monitoring service", "author": { diff --git a/public/app/plugins/datasource/cloudwatch/query_ctrl.ts b/public/app/plugins/datasource/cloudwatch/query_ctrl.ts deleted file mode 100644 index d1fe69e626e..00000000000 --- a/public/app/plugins/datasource/cloudwatch/query_ctrl.ts +++ /dev/null @@ -1,15 +0,0 @@ -import './query_parameter_ctrl'; -import { QueryCtrl } from 'app/plugins/sdk'; -import { auto } from 'angular'; - -export class CloudWatchQueryCtrl extends QueryCtrl { - static templateUrl = 'partials/query.editor.html'; - - aliasSyntax: string; - - /** @ngInject */ - constructor($scope: any, $injector: auto.IInjectorService) { - super($scope, $injector); - this.aliasSyntax = '{{metric}} {{stat}} {{namespace}} {{region}} {{}}'; - } -} diff --git a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts index 1a7ffa6cacc..6c3806dd91a 100644 --- a/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts +++ b/public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts @@ -1,5 +1,6 @@ import '../datasource'; import CloudWatchDatasource from '../datasource'; +import * as redux from 'app/store/store'; import { dateMath } from '@grafana/data'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { CustomVariable } from 'app/features/templating/all'; @@ -12,6 +13,7 @@ import { TimeSrv } from 'app/features/dashboard/services/TimeSrv'; describe('CloudWatchDatasource', () => { const instanceSettings = { jsonData: { defaultRegion: 'us-east-1' }, + name: 'TestDatasource', } as DataSourceInstanceSettings; const templateSrv = new TemplateSrv(); @@ -45,6 +47,7 @@ describe('CloudWatchDatasource', () => { rangeRaw: { from: 1483228800, to: 1483232400 }, targets: [ { + expression: '', refId: 'A', region: 'us-east-1', namespace: 'AWS/EC2', @@ -90,7 +93,7 @@ describe('CloudWatchDatasource', () => { const params = requestParams.queries[0]; expect(params.namespace).toBe(query.targets[0].namespace); expect(params.metricName).toBe(query.targets[0].metricName); - expect(params.dimensions['InstanceId']).toBe('i-12345678'); + expect(params.dimensions['InstanceId']).toStrictEqual(['i-12345678']); expect(params.statistics).toEqual(query.targets[0].statistics); expect(params.period).toBe(query.targets[0].period); done(); @@ -164,6 +167,142 @@ describe('CloudWatchDatasource', () => { done(); }); }); + + describe('a correct cloudwatch url should be built for each time series in the response', () => { + beforeEach(() => { + ctx.backendSrv.datasourceRequest = jest.fn(params => { + requestParams = params.data; + return Promise.resolve({ data: response }); + }); + }); + + it('should be built correctly if theres one search expressions returned in meta for a given query row', done => { + response.results['A'].meta.gmdMeta = [{ Expression: `REMOVE_EMPTY(SEARCH('some expression'))` }]; + ctx.ds.query(query).then((result: any) => { + expect(result.data[0].name).toBe(response.results.A.series[0].name); + expect(result.data[0].fields[0].config.links[0].title).toBe('View in CloudWatch console'); + expect(decodeURIComponent(result.data[0].fields[0].config.links[0].url)).toContain( + `region=us-east-1#metricsV2:graph={"view":"timeSeries","stacked":false,"title":"A","start":"2016-12-31T15:00:00.000Z","end":"2016-12-31T16:00:00.000Z","region":"us-east-1","metrics":[{"expression":"REMOVE_EMPTY(SEARCH(\'some expression\'))"}]}` + ); + done(); + }); + }); + + it('should be built correctly if theres two search expressions returned in meta for a given query row', done => { + response.results['A'].meta.gmdMeta = [ + { Expression: `REMOVE_EMPTY(SEARCH('first expression'))` }, + { Expression: `REMOVE_EMPTY(SEARCH('second expression'))` }, + ]; + ctx.ds.query(query).then((result: any) => { + expect(result.data[0].name).toBe(response.results.A.series[0].name); + expect(result.data[0].fields[0].config.links[0].title).toBe('View in CloudWatch console'); + expect(decodeURIComponent(result.data[0].fields[0].config.links[0].url)).toContain( + `region=us-east-1#metricsV2:graph={"view":"timeSeries","stacked":false,"title":"A","start":"2016-12-31T15:00:00.000Z","end":"2016-12-31T16:00:00.000Z","region":"us-east-1","metrics":[{"expression":"REMOVE_EMPTY(SEARCH(\'first expression\'))"},{"expression":"REMOVE_EMPTY(SEARCH(\'second expression\'))"}]}` + ); + done(); + }); + }); + + it('should be built correctly if the query is a metric stat query', done => { + response.results['A'].meta.gmdMeta = []; + ctx.ds.query(query).then((result: any) => { + expect(result.data[0].name).toBe(response.results.A.series[0].name); + expect(result.data[0].fields[0].config.links[0].title).toBe('View in CloudWatch console'); + expect(decodeURIComponent(result.data[0].fields[0].config.links[0].url)).toContain( + `region=us-east-1#metricsV2:graph={\"view\":\"timeSeries\",\"stacked\":false,\"title\":\"A\",\"start\":\"2016-12-31T15:00:00.000Z\",\"end\":\"2016-12-31T16:00:00.000Z\",\"region\":\"us-east-1\",\"metrics\":[[\"AWS/EC2\",\"CPUUtilization\",\"InstanceId\",\"i-12345678\",{\"stat\":\"Average\",\"period\":\"300\"}]]}` + ); + done(); + }); + }); + + it('should not be added at all if query is a math expression', done => { + query.targets[0].expression = 'a * 2'; + response.results['A'].meta.searchExpressions = []; + ctx.ds.query(query).then((result: any) => { + expect(result.data[0].fields[0].config.links).toBeUndefined(); + done(); + }); + }); + }); + + describe('and throttling exception is thrown', () => { + const partialQuery = { + namespace: 'AWS/EC2', + metricName: 'CPUUtilization', + dimensions: { + InstanceId: 'i-12345678', + }, + statistics: ['Average'], + period: '300', + expression: '', + }; + + const query = { + range: defaultTimeRange, + rangeRaw: { from: 1483228800, to: 1483232400 }, + targets: [ + { ...partialQuery, refId: 'A', region: 'us-east-1' }, + { ...partialQuery, refId: 'B', region: 'us-east-2' }, + { ...partialQuery, refId: 'C', region: 'us-east-1' }, + { ...partialQuery, refId: 'D', region: 'us-east-2' }, + { ...partialQuery, refId: 'E', region: 'eu-north-1' }, + ], + }; + + const backendErrorResponse = { + data: { + message: 'Throttling: exception', + results: { + A: { + error: 'Throttling: exception', + refId: 'A', + meta: {}, + }, + B: { + error: 'Throttling: exception', + refId: 'B', + meta: {}, + }, + C: { + error: 'Throttling: exception', + refId: 'C', + meta: {}, + }, + D: { + error: 'Throttling: exception', + refId: 'D', + meta: {}, + }, + E: { + error: 'Throttling: exception', + refId: 'E', + meta: {}, + }, + }, + }, + }; + + beforeEach(() => { + redux.setStore({ + dispatch: jest.fn(), + }); + + ctx.backendSrv.datasourceRequest = jest.fn(() => { + return Promise.reject(backendErrorResponse); + }); + }); + + it('should display one alert error message per region+datasource combination', done => { + const memoizedDebounceSpy = jest.spyOn(ctx.ds, 'debouncedAlert'); + ctx.ds.query(query).catch(() => { + expect(memoizedDebounceSpy).toHaveBeenCalledWith('TestDatasource', 'us-east-1'); + expect(memoizedDebounceSpy).toHaveBeenCalledWith('TestDatasource', 'us-east-2'); + expect(memoizedDebounceSpy).toHaveBeenCalledWith('TestDatasource', 'eu-north-1'); + expect(memoizedDebounceSpy).toBeCalledTimes(3); + done(); + }); + }); + }); }); describe('When query region is "default"', () => { @@ -308,6 +447,21 @@ describe('CloudWatchDatasource', () => { }, {} as any ), + new CustomVariable( + { + name: 'var4', + options: [ + { selected: true, value: 'var4-foo' }, + { selected: false, value: 'var4-bar' }, + { selected: true, value: 'var4-baz' }, + ], + current: { + value: ['var4-foo', 'var4-baz'], + }, + multi: true, + }, + {} as any + ), ]); ctx.backendSrv.datasourceRequest = jest.fn(params => { @@ -336,12 +490,12 @@ describe('CloudWatchDatasource', () => { }; ctx.ds.query(query).then(() => { - expect(requestParams.queries[0].dimensions['dim2']).toBe('var2-foo'); + expect(requestParams.queries[0].dimensions['dim2']).toStrictEqual(['var2-foo']); done(); }); }); - it('should generate the correct query for multilple template variables', done => { + it('should generate the correct query in the case of one multilple template variables', done => { const query = { range: defaultTimeRange, rangeRaw: { from: 1483228800, to: 1483232400 }, @@ -367,17 +521,14 @@ describe('CloudWatchDatasource', () => { }; ctx.ds.query(query).then(() => { - expect(requestParams.queries[0].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[0].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[0].dimensions['dim3']).toBe('var3-foo'); - expect(requestParams.queries[1].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[1].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[1].dimensions['dim3']).toBe('var3-baz'); + expect(requestParams.queries[0].dimensions['dim1']).toStrictEqual(['var1-foo']); + expect(requestParams.queries[0].dimensions['dim2']).toStrictEqual(['var2-foo']); + expect(requestParams.queries[0].dimensions['dim3']).toStrictEqual(['var3-foo', 'var3-baz']); done(); }); }); - it('should generate the correct query for multilple template variables, lack scopedVars', done => { + it('should generate the correct query in the case of multilple multi template variables', done => { const query = { range: defaultTimeRange, rangeRaw: { from: 1483228800, to: 1483232400 }, @@ -389,37 +540,30 @@ describe('CloudWatchDatasource', () => { metricName: 'TestMetricName', dimensions: { dim1: '[[var1]]', - dim2: '[[var2]]', dim3: '[[var3]]', + dim4: '[[var4]]', }, statistics: ['Average'], period: 300, }, ], - scopedVars: { - var1: { selected: true, value: 'var1-foo' }, - }, }; ctx.ds.query(query).then(() => { - expect(requestParams.queries[0].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[0].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[0].dimensions['dim3']).toBe('var3-foo'); - expect(requestParams.queries[1].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[1].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[1].dimensions['dim3']).toBe('var3-baz'); + expect(requestParams.queries[0].dimensions['dim1']).toStrictEqual(['var1-foo']); + expect(requestParams.queries[0].dimensions['dim3']).toStrictEqual(['var3-foo', 'var3-baz']); + expect(requestParams.queries[0].dimensions['dim4']).toStrictEqual(['var4-foo', 'var4-baz']); done(); }); }); - it('should generate the correct query for multilple template variables with expression', done => { - const query: any = { + it('should generate the correct query for multilple template variables, lack scopedVars', done => { + const query = { range: defaultTimeRange, rangeRaw: { from: 1483228800, to: 1483232400 }, targets: [ { refId: 'A', - id: 'id1', region: 'us-east-1', namespace: 'TestNamespace', metricName: 'TestMetricName', @@ -430,39 +574,17 @@ describe('CloudWatchDatasource', () => { }, statistics: ['Average'], period: 300, - expression: '', - }, - { - refId: 'B', - id: 'id2', - expression: 'METRICS("id1") * 2', - dimensions: { - // garbage data for fail test - dim1: '[[var1]]', - dim2: '[[var2]]', - dim3: '[[var3]]', - }, - statistics: [], // dummy }, ], scopedVars: { var1: { selected: true, value: 'var1-foo' }, - var2: { selected: true, value: 'var2-foo' }, }, }; ctx.ds.query(query).then(() => { - expect(requestParams.queries.length).toBe(3); - expect(requestParams.queries[0].id).toMatch(/^id1.*/); - expect(requestParams.queries[0].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[0].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[0].dimensions['dim3']).toBe('var3-foo'); - expect(requestParams.queries[1].id).toMatch(/^id1.*/); - expect(requestParams.queries[1].dimensions['dim1']).toBe('var1-foo'); - expect(requestParams.queries[1].dimensions['dim2']).toBe('var2-foo'); - expect(requestParams.queries[1].dimensions['dim3']).toBe('var3-baz'); - expect(requestParams.queries[2].id).toMatch(/^id2.*/); - expect(requestParams.queries[2].expression).toBe('METRICS("id1") * 2'); + expect(requestParams.queries[0].dimensions['dim1']).toStrictEqual(['var1-foo']); + expect(requestParams.queries[0].dimensions['dim2']).toStrictEqual(['var2-foo']); + expect(requestParams.queries[0].dimensions['dim3']).toStrictEqual(['var3-foo', 'var3-baz']); done(); }); }); @@ -471,9 +593,9 @@ describe('CloudWatchDatasource', () => { function describeMetricFindQuery(query: any, func: any) { describe('metricFindQuery ' + query, () => { const scenario: any = {}; - scenario.setup = (setupCallback: any) => { - beforeEach(() => { - setupCallback(); + scenario.setup = async (setupCallback: any) => { + beforeEach(async () => { + await setupCallback(); ctx.backendSrv.datasourceRequest = jest.fn(args => { scenario.request = args.data; return Promise.resolve({ data: scenario.requestResponse }); @@ -488,8 +610,8 @@ describe('CloudWatchDatasource', () => { }); } - describeMetricFindQuery('regions()', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('regions()', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -506,8 +628,8 @@ describe('CloudWatchDatasource', () => { }); }); - describeMetricFindQuery('namespaces()', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('namespaces()', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -524,8 +646,8 @@ describe('CloudWatchDatasource', () => { }); }); - describeMetricFindQuery('metrics(AWS/EC2)', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('metrics(AWS/EC2, us-east-2)', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -542,8 +664,8 @@ describe('CloudWatchDatasource', () => { }); }); - describeMetricFindQuery('dimension_keys(AWS/EC2)', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('dimension_keys(AWS/EC2)', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -554,14 +676,15 @@ describe('CloudWatchDatasource', () => { }); it('should call __GetDimensions and return result', () => { + console.log({ a: scenario.requestResponse.results }); expect(scenario.result[0].text).toBe('InstanceId'); expect(scenario.request.queries[0].type).toBe('metricFindQuery'); expect(scenario.request.queries[0].subtype).toBe('dimension_keys'); }); }); - describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization,InstanceId)', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('dimension_values(us-east-1,AWS/EC2,CPUUtilization,InstanceId)', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -578,8 +701,8 @@ describe('CloudWatchDatasource', () => { }); }); - describeMetricFindQuery('dimension_values(default,AWS/EC2,CPUUtilization,InstanceId)', (scenario: any) => { - scenario.setup(() => { + describeMetricFindQuery('dimension_values(default,AWS/EC2,CPUUtilization,InstanceId)', async (scenario: any) => { + await scenario.setup(() => { scenario.requestResponse = { results: { metricFindQuery: { @@ -596,32 +719,35 @@ describe('CloudWatchDatasource', () => { }); }); - describeMetricFindQuery('resource_arns(default,ec2:instance,{"environment":["production"]})', (scenario: any) => { - scenario.setup(() => { - scenario.requestResponse = { - results: { - metricFindQuery: { - tables: [ - { - rows: [ - [ - 'arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567', - 'arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321', + describeMetricFindQuery( + 'resource_arns(default,ec2:instance,{"environment":["production"]})', + async (scenario: any) => { + await scenario.setup(() => { + scenario.requestResponse = { + results: { + metricFindQuery: { + tables: [ + { + rows: [ + [ + 'arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567', + 'arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321', + ], ], - ], - }, - ], + }, + ], + }, }, - }, - }; - }); + }; + }); - it('should call __ListMetrics and return result', () => { - expect(scenario.result[0].text).toContain('arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567'); - expect(scenario.request.queries[0].type).toBe('metricFindQuery'); - expect(scenario.request.queries[0].subtype).toBe('resource_arns'); - }); - }); + it('should call __ListMetrics and return result', () => { + expect(scenario.result[0].text).toContain('arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567'); + expect(scenario.request.queries[0].type).toBe('metricFindQuery'); + expect(scenario.request.queries[0].subtype).toBe('resource_arns'); + }); + } + ); it('should caclculate the correct period', () => { const hourSec = 60 * 60; diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index c18cbd528bb..170f30a6318 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -1,12 +1,29 @@ -import { DataQuery } from '@grafana/data'; +import { DataQuery, SelectableValue, DataSourceJsonData } from '@grafana/data'; export interface CloudWatchQuery extends DataQuery { id: string; region: string; namespace: string; metricName: string; - dimensions: { [key: string]: string }; + dimensions: { [key: string]: string | string[] }; statistics: string[]; period: string; expression: string; + alias: string; + highResolution: boolean; + matchExact: boolean; +} + +export type SelectableStrings = Array>; + +export interface CloudWatchJsonData extends DataSourceJsonData { + timeField?: string; + assumeRoleArn?: string; + database?: string; + customMetricsNamespaces?: string; +} + +export interface CloudWatchSecureJsonData { + accessKey: string; + secretKey: string; } diff --git a/public/app/types/appNotifications.ts b/public/app/types/appNotifications.ts index b9b43e72683..7e5a3dba599 100644 --- a/public/app/types/appNotifications.ts +++ b/public/app/types/appNotifications.ts @@ -4,6 +4,7 @@ export interface AppNotification { icon: string; title: string; text: string; + component?: React.ReactElement; timeout: AppNotificationTimeout; } diff --git a/scripts/go/go.mod b/scripts/go/go.mod index 3345d5efe9a..332724133c5 100644 --- a/scripts/go/go.mod +++ b/scripts/go/go.mod @@ -3,13 +3,12 @@ module github.com/grafana/grafana/scripts/go go 1.13 require ( - github.com/golangci/golangci-lint v1.20.0 + github.com/golangci/golangci-lint v1.19.2-0.20191006164544-7577d548a389 github.com/mgechev/revive v0.0.0-20190917153825-40564c5052ae - github.com/securego/gosec v0.0.0-20191002120514-e680875ea14d + github.com/securego/gosec v0.0.0-20190912120752-140048b2a218 github.com/unknwon/bra v0.0.0-20190805204333-bb0929b6cca0 github.com/unknwon/com v1.0.1 // indirect github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3 // indirect github.com/urfave/cli v1.20.0 // indirect - golang.org/x/tools v0.0.0-20191015150414-f936694f27bf // indirect gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect )