From 4c654ddb76a5f554b6748b149d2c770bfba83e70 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 25 Oct 2022 14:00:54 +0200 Subject: [PATCH] Cloudwatch: Refactor metrics resource request (#57424) * refactor metrics request * Update pkg/tsdb/cloudwatch/routes/dimension_keys_test.go Co-authored-by: Shirley <4163034+fridgepoet@users.noreply.github.com> * return metric struct value intead of pointer * make it possible to test hard coded metrics service * test all paths in route * fix broken test * fix one more broken test * add integration test Co-authored-by: Shirley <4163034+fridgepoet@users.noreply.github.com> --- pkg/tsdb/cloudwatch/cloudwatch_test.go | 36 ++++++ .../cloudwatch/mocks/list_metrics_service.go | 5 +- pkg/tsdb/cloudwatch/models/metric_types.go | 2 +- .../models/request/dimension_keys_request.go | 4 +- pkg/tsdb/cloudwatch/models/request/metrics.go | 42 +++++++ .../cloudwatch/models/request/metrics_test.go | 48 ++++++++ pkg/tsdb/cloudwatch/models/request/utils.go | 9 ++ pkg/tsdb/cloudwatch/models/types.go | 5 + pkg/tsdb/cloudwatch/resource_handler.go | 3 +- pkg/tsdb/cloudwatch/routes/dimension_keys.go | 2 +- .../cloudwatch/routes/dimension_keys_test.go | 108 ++++++++++-------- pkg/tsdb/cloudwatch/routes/metrics.go | 44 +++++++ pkg/tsdb/cloudwatch/routes/metrics_test.go | 88 ++++++++++++++ .../cloudwatch/services/hardcoded_metrics.go | 43 +++++++ .../services/hardcoded_metrics_test.go | 39 +++++++ pkg/tsdb/cloudwatch/services/list_metrics.go | 29 +++-- .../cloudwatch/services/list_metrics_test.go | 17 --- .../plugins/datasource/cloudwatch/api.test.ts | 1 + .../app/plugins/datasource/cloudwatch/api.ts | 11 +- .../datasource/cloudwatch/datasource.test.ts | 8 +- .../plugins/datasource/cloudwatch/types.ts | 5 + 21 files changed, 458 insertions(+), 91 deletions(-) create mode 100644 pkg/tsdb/cloudwatch/models/request/metrics.go create mode 100644 pkg/tsdb/cloudwatch/models/request/metrics_test.go create mode 100644 pkg/tsdb/cloudwatch/routes/metrics.go create mode 100644 pkg/tsdb/cloudwatch/routes/metrics_test.go create mode 100644 pkg/tsdb/cloudwatch/services/hardcoded_metrics.go create mode 100644 pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go diff --git a/pkg/tsdb/cloudwatch/cloudwatch_test.go b/pkg/tsdb/cloudwatch/cloudwatch_test.go index 32d6c9b9f01..08e95fef1d8 100644 --- a/pkg/tsdb/cloudwatch/cloudwatch_test.go +++ b/pkg/tsdb/cloudwatch/cloudwatch_test.go @@ -602,6 +602,42 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) { require.Nil(t, err) assert.Equal(t, []string{"ClientId", "DomainName"}, res) }) + + t.Run("Should handle custom namespace metrics query and return metrics from api", func(t *testing.T) { + pageLimit := 3 + api = mocks.FakeMetricsAPI{Metrics: []*cloudwatch.Metric{ + {MetricName: aws.String("Test_MetricName1"), Namespace: aws.String("AWS/EC2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}, {Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName2"), Namespace: aws.String("AWS/EC2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName3"), Namespace: aws.String("AWS/ECS"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName10"), Namespace: aws.String("AWS/ECS"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}, {Name: aws.String("Test_DimensionName5")}}}, + {MetricName: aws.String("Test_MetricName4"), Namespace: aws.String("AWS/ECS"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName2")}}}, + {MetricName: aws.String("Test_MetricName5"), Namespace: aws.String("AWS/Redshift"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName6"), Namespace: aws.String("AWS/Redshift"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + {MetricName: aws.String("Test_MetricName7"), Namespace: aws.String("AWS/EC2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}}}, + {MetricName: aws.String("Test_MetricName8"), Namespace: aws.String("AWS/EC2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName4")}}}, + {MetricName: aws.String("Test_MetricName9"), Namespace: aws.String("AWS/EC2"), Dimensions: []*cloudwatch.Dimension{{Name: aws.String("Test_DimensionName1")}}}, + }, MetricsPerPage: 2} + executor := newExecutor(im, &setting.Cfg{AWSListMetricsPageLimit: pageLimit}, &fakeSessionCache{}, featuremgmt.WithFeatures()) + + req := &backend.CallResourceRequest{ + Method: "GET", + Path: `/metrics?region=us-east-2&namespace=custom-namespace`, + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0}, + PluginID: "cloudwatch", + }, + } + err := executor.CallResource(context.Background(), req, sender) + + require.NoError(t, err) + sent := sender.Response + require.NotNil(t, sent) + require.Equal(t, http.StatusOK, sent.Status) + res := []models.Metric{} + err = json.Unmarshal(sent.Body, &res) + require.Nil(t, err) + assert.Equal(t, []models.Metric{{Name: "Test_MetricName1", Namespace: "AWS/EC2"}, {Name: "Test_MetricName2", Namespace: "AWS/EC2"}, {Name: "Test_MetricName3", Namespace: "AWS/ECS"}, {Name: "Test_MetricName10", Namespace: "AWS/ECS"}, {Name: "Test_MetricName4", Namespace: "AWS/ECS"}, {Name: "Test_MetricName5", Namespace: "AWS/Redshift"}}, res) + }) } func stringsToSuggestData(values []string) []suggestData { diff --git a/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go index 0be8f18e884..7bd44ca004b 100644 --- a/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go +++ b/pkg/tsdb/cloudwatch/mocks/list_metrics_service.go @@ -1,6 +1,7 @@ package mocks import ( + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/request" "github.com/stretchr/testify/mock" ) @@ -27,8 +28,8 @@ func (a *ListMetricsServiceMock) GetDimensionKeysByNamespace(string) ([]string, return args.Get(0).([]string), args.Error(1) } -func (a *ListMetricsServiceMock) GetHardCodedDimensionKeysByNamespace(string) ([]string, error) { +func (a *ListMetricsServiceMock) GetMetricsByNamespace(namespace string) ([]models.Metric, error) { args := a.Called() - return args.Get(0).([]string), args.Error(1) + return args.Get(0).([]models.Metric), args.Error(1) } diff --git a/pkg/tsdb/cloudwatch/models/metric_types.go b/pkg/tsdb/cloudwatch/models/metric_types.go index 4eb63536b1e..ec6c1ed6087 100644 --- a/pkg/tsdb/cloudwatch/models/metric_types.go +++ b/pkg/tsdb/cloudwatch/models/metric_types.go @@ -7,9 +7,9 @@ import ( type ListMetricsProvider interface { GetDimensionKeysByDimensionFilter(*request.DimensionKeysRequest) ([]string, error) - GetHardCodedDimensionKeysByNamespace(string) ([]string, error) GetDimensionKeysByNamespace(string) ([]string, error) GetDimensionValuesByDimensionFilter(*request.DimensionValuesRequest) ([]string, error) + GetMetricsByNamespace(namespace string) ([]Metric, error) } type MetricsClientProvider interface { diff --git a/pkg/tsdb/cloudwatch/models/request/dimension_keys_request.go b/pkg/tsdb/cloudwatch/models/request/dimension_keys_request.go index 0416e767ae9..7713cdb6ce8 100644 --- a/pkg/tsdb/cloudwatch/models/request/dimension_keys_request.go +++ b/pkg/tsdb/cloudwatch/models/request/dimension_keys_request.go @@ -2,8 +2,6 @@ package request import ( "net/url" - - "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" ) type DimensionKeysRequestType uint32 @@ -22,7 +20,7 @@ type DimensionKeysRequest struct { } func (q *DimensionKeysRequest) Type() DimensionKeysRequestType { - if _, exist := constants.NamespaceMetricsMap[q.Namespace]; !exist { + if isCustomNamespace(q.Namespace) { return CustomMetricDimensionKeysRequest } diff --git a/pkg/tsdb/cloudwatch/models/request/metrics.go b/pkg/tsdb/cloudwatch/models/request/metrics.go new file mode 100644 index 00000000000..a4f2e65fa45 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/request/metrics.go @@ -0,0 +1,42 @@ +package request + +import ( + "net/url" +) + +type MetricsRequestType uint32 + +const ( + MetricsByNamespaceRequestType MetricsRequestType = iota + AllMetricsRequestType + CustomNamespaceRequestType +) + +type MetricsRequest struct { + *ResourceRequest + Namespace string +} + +func GetMetricsRequest(parameters url.Values) (*MetricsRequest, error) { + resourceRequest, err := getResourceRequest(parameters) + if err != nil { + return nil, err + } + + return &MetricsRequest{ + ResourceRequest: resourceRequest, + Namespace: parameters.Get("namespace"), + }, nil +} + +func (r *MetricsRequest) Type() MetricsRequestType { + if r.Namespace == "" { + return AllMetricsRequestType + } + + if isCustomNamespace(r.Namespace) { + return CustomNamespaceRequestType + } + + return MetricsByNamespaceRequestType +} diff --git a/pkg/tsdb/cloudwatch/models/request/metrics_test.go b/pkg/tsdb/cloudwatch/models/request/metrics_test.go new file mode 100644 index 00000000000..0e1038a6669 --- /dev/null +++ b/pkg/tsdb/cloudwatch/models/request/metrics_test.go @@ -0,0 +1,48 @@ +package request + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMetricsRequest(t *testing.T) { + t.Run("Should parse parameters", func(t *testing.T) { + request, err := GetMetricsRequest(map[string][]string{"region": {"us-east-1"}, "namespace": {"AWS/EC2"}}) + require.NoError(t, err) + assert.Equal(t, "us-east-1", request.Region) + assert.Equal(t, "AWS/EC2", request.Namespace) + }) + + tests := []struct { + reqType MetricsRequestType + params url.Values + }{ + { + params: map[string][]string{"region": {"us-east-1"}, "namespace": {"AWS/EC2"}}, + reqType: MetricsByNamespaceRequestType, + }, + { + params: map[string][]string{"region": {"us-east-1"}}, + reqType: AllMetricsRequestType, + }, + { + params: map[string][]string{"region": {"us-east-1"}, "namespace": {""}}, + reqType: AllMetricsRequestType, + }, + { + params: map[string][]string{"region": {"us-east-1"}, "namespace": {"custom-namespace"}}, + reqType: CustomNamespaceRequestType, + }, + } + + for _, tc := range tests { + t.Run("Should resolve the correct type", func(t *testing.T) { + request, err := GetMetricsRequest(tc.params) + require.NoError(t, err) + assert.Equal(t, tc.reqType, request.Type()) + }) + } +} diff --git a/pkg/tsdb/cloudwatch/models/request/utils.go b/pkg/tsdb/cloudwatch/models/request/utils.go index 6e12506d49f..4057a5f095d 100644 --- a/pkg/tsdb/cloudwatch/models/request/utils.go +++ b/pkg/tsdb/cloudwatch/models/request/utils.go @@ -3,6 +3,8 @@ package request import ( "encoding/json" "fmt" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" ) func parseDimensionFilter(dimensionFilter string) ([]*Dimension, error) { @@ -42,3 +44,10 @@ func parseDimensionFilter(dimensionFilter string) ([]*Dimension, error) { return dimensions, nil } + +func isCustomNamespace(namespace string) bool { + if _, ok := constants.NamespaceMetricsMap[namespace]; ok { + return false + } + return true +} diff --git a/pkg/tsdb/cloudwatch/models/types.go b/pkg/tsdb/cloudwatch/models/types.go index 9d098f9341e..a028658aa47 100644 --- a/pkg/tsdb/cloudwatch/models/types.go +++ b/pkg/tsdb/cloudwatch/models/types.go @@ -34,3 +34,8 @@ type metricStatMeta struct { Period int `json:"period"` Label string `json:"label,omitempty"` } + +type Metric struct { + Name string `json:"name"` + Namespace string `json:"namespace"` +} diff --git a/pkg/tsdb/cloudwatch/resource_handler.go b/pkg/tsdb/cloudwatch/resource_handler.go index 95b2aa66615..95d37c750e9 100644 --- a/pkg/tsdb/cloudwatch/resource_handler.go +++ b/pkg/tsdb/cloudwatch/resource_handler.go @@ -16,13 +16,12 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux { mux := http.NewServeMux() mux.HandleFunc("/regions", handleResourceReq(e.handleGetRegions)) mux.HandleFunc("/namespaces", handleResourceReq(e.handleGetNamespaces)) - mux.HandleFunc("/metrics", handleResourceReq(e.handleGetMetrics)) - mux.HandleFunc("/all-metrics", handleResourceReq(e.handleGetAllMetrics)) mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds)) mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute)) mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns)) mux.HandleFunc("/log-groups", handleResourceReq(e.handleGetLogGroups)) mux.HandleFunc("/all-log-groups", handleResourceReq(e.handleGetAllLogGroups)) + mux.HandleFunc("/metrics", routes.ResourceRequestMiddleware(routes.MetricsHandler, e.getClients)) mux.HandleFunc("/dimension-values", routes.ResourceRequestMiddleware(routes.DimensionValuesHandler, e.getClients)) mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, e.getClients)) return mux diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys.go b/pkg/tsdb/cloudwatch/routes/dimension_keys.go index 743ac46b45e..20e98ab09e9 100644 --- a/pkg/tsdb/cloudwatch/routes/dimension_keys.go +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys.go @@ -25,7 +25,7 @@ func DimensionKeysHandler(pluginCtx backend.PluginContext, clientFactory models. dimensionKeys := []string{} switch dimensionKeysRequest.Type() { case request.StandardDimensionKeysRequest: - dimensionKeys, err = service.GetHardCodedDimensionKeysByNamespace(dimensionKeysRequest.Namespace) + dimensionKeys, err = services.GetHardCodedDimensionKeysByNamespace(dimensionKeysRequest.Namespace) case request.FilterDimensionKeysRequest: dimensionKeys, err = service.GetDimensionKeysByDimensionFilter(dimensionKeysRequest) case request.CustomMetricDimensionKeysRequest: diff --git a/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go index f68b9258798..b96212dc8a4 100644 --- a/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go +++ b/pkg/tsdb/cloudwatch/routes/dimension_keys_test.go @@ -1,6 +1,7 @@ package routes import ( + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -9,57 +10,72 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_DimensionKeys_Route(t *testing.T) { - tests := []struct { - url string - methodName string - requestType string - }{ - { - url: "/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization", - methodName: "GetHardCodedDimensionKeysByNamespace", - requestType: "StandardDimensionKeysRequest"}, - { - url: `/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization&dimensionFilters={"NodeID":["Shared"],"stage":["QueryCommit"]}`, - methodName: "GetDimensionKeysByDimensionFilter", - requestType: "FilterDimensionKeysRequest"}, - { - url: `/dimension-keys?region=us-east-2&namespace=customNamespace&metricName=CPUUtilization`, - methodName: "GetDimensionKeysByNamespace", - requestType: "CustomMetricDimensionKeysRequest"}, - } + t.Run("calls FilterDimensionKeysRequest when a StandardDimensionKeysRequest is passed", func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On("GetDimensionKeysByDimensionFilter").Return([]string{}, nil) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", `/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization&dimensionFilters={"NodeID":["Shared"],"stage":["QueryCommit"]}`, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + mockListMetricsService.AssertNumberOfCalls(t, "GetDimensionKeysByDimensionFilter", 1) + }) - for _, tc := range tests { - t.Run(fmt.Sprintf("calls %s when a StandardDimensionKeysRequest is passed", tc.requestType), func(t *testing.T) { - mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On(tc.methodName).Return([]string{}, nil) - newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { - return &mockListMetricsService, nil - } - rr := httptest.NewRecorder() - req := httptest.NewRequest("GET", tc.url, nil) - handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) - handler.ServeHTTP(rr, req) - mockListMetricsService.AssertNumberOfCalls(t, tc.methodName, 1) - }) - } + t.Run("calls GetDimensionKeysByNamespace when a CustomMetricDimensionKeysRequest is passed", func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On("GetDimensionKeysByNamespace").Return([]string{}, nil) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", `/dimension-keys?region=us-east-2&namespace=custom&metricName=CPUUtilization`, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + mockListMetricsService.AssertNumberOfCalls(t, "GetDimensionKeysByNamespace", 1) + }) - for _, tc := range tests { - t.Run(fmt.Sprintf("return 500 if %s returns an error", tc.requestType), func(t *testing.T) { - mockListMetricsService := mocks.ListMetricsServiceMock{} - mockListMetricsService.On(tc.methodName).Return([]string{}, fmt.Errorf("some error")) - newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { - return &mockListMetricsService, nil - } - rr := httptest.NewRecorder() - req := httptest.NewRequest("GET", tc.url, nil) - handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) - handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusInternalServerError, rr.Code) - assert.Equal(t, `{"Message":"error in DimensionKeyHandler: some error","Error":"some error","StatusCode":500}`, rr.Body.String()) + t.Run("calls GetHardCodedDimensionKeysByNamespace when a StandardDimensionKeysRequest is passed", func(t *testing.T) { + origGetHardCodedDimensionKeysByNamespace := services.GetHardCodedDimensionKeysByNamespace + t.Cleanup(func() { + services.GetHardCodedDimensionKeysByNamespace = origGetHardCodedDimensionKeysByNamespace }) - } + haveBeenCalled := false + usedNamespace := "" + services.GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]string, error) { + haveBeenCalled = true + usedNamespace = namespace + return []string{}, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + res := []models.Metric{} + err := json.Unmarshal(rr.Body.Bytes(), &res) + require.Nil(t, err) + assert.True(t, haveBeenCalled) + assert.Equal(t, "AWS/EC2", usedNamespace) + }) + + t.Run("return 500 if GetDimensionKeysByDimensionFilter returns an error", func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On("GetDimensionKeysByDimensionFilter").Return([]string{}, fmt.Errorf("some error")) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", `/dimension-keys?region=us-east-2&namespace=AWS/EC2&metricName=CPUUtilization&dimensionFilters={"NodeID":["Shared"],"stage":["QueryCommit"]}`, nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(DimensionKeysHandler, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, `{"Message":"error in DimensionKeyHandler: some error","Error":"some error","StatusCode":500}`, rr.Body.String()) + }) } diff --git a/pkg/tsdb/cloudwatch/routes/metrics.go b/pkg/tsdb/cloudwatch/routes/metrics.go new file mode 100644 index 00000000000..f685c9071a4 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/metrics.go @@ -0,0 +1,44 @@ +package routes + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/request" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" +) + +func MetricsHandler(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) { + metricsRequest, err := request.GetMetricsRequest(parameters) + if err != nil { + return nil, models.NewHttpError("error in MetricsHandler", http.StatusBadRequest, err) + } + + service, err := newListMetricsService(pluginCtx, clientFactory, metricsRequest.Region) + if err != nil { + return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) + } + + var metrics []models.Metric + switch metricsRequest.Type() { + case request.AllMetricsRequestType: + metrics = services.GetAllHardCodedMetrics() + case request.MetricsByNamespaceRequestType: + metrics, err = services.GetHardCodedMetricsByNamespace(metricsRequest.Namespace) + case request.CustomNamespaceRequestType: + metrics, err = service.GetMetricsByNamespace(metricsRequest.Namespace) + } + if err != nil { + return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) + } + + metricsResponse, err := json.Marshal(metrics) + if err != nil { + return nil, models.NewHttpError("error in MetricsHandler", http.StatusInternalServerError, err) + } + + return metricsResponse, nil +} diff --git a/pkg/tsdb/cloudwatch/routes/metrics_test.go b/pkg/tsdb/cloudwatch/routes/metrics_test.go new file mode 100644 index 00000000000..4a8e2a885b7 --- /dev/null +++ b/pkg/tsdb/cloudwatch/routes/metrics_test.go @@ -0,0 +1,88 @@ +package routes + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/services" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_Metrics_Route(t *testing.T) { + t.Run("calls GetMetricsByNamespace when a CustomNamespaceRequestType is passed", func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On("GetMetricsByNamespace").Return([]models.Metric{}, nil) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/metrics?region=us-east-2&namespace=customNamespace", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, nil)) + handler.ServeHTTP(rr, req) + mockListMetricsService.AssertNumberOfCalls(t, "GetMetricsByNamespace", 1) + }) + + t.Run("calls GetAllHardCodedMetrics when a AllMetricsRequestType is passed", func(t *testing.T) { + origGetAllHardCodedMetrics := services.GetAllHardCodedMetrics + t.Cleanup(func() { + services.GetAllHardCodedMetrics = origGetAllHardCodedMetrics + }) + haveBeenCalled := false + services.GetAllHardCodedMetrics = func() []models.Metric { + haveBeenCalled = true + return []models.Metric{} + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/metrics?region=us-east-2", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, nil)) + handler.ServeHTTP(rr, req) + res := []models.Metric{} + err := json.Unmarshal(rr.Body.Bytes(), &res) + require.Nil(t, err) + assert.True(t, haveBeenCalled) + }) + + t.Run("calls GetHardCodedMetricsByNamespace when a MetricsByNamespaceRequestType is passed", func(t *testing.T) { + origGetHardCodedMetricsByNamespace := services.GetHardCodedMetricsByNamespace + t.Cleanup(func() { + services.GetHardCodedMetricsByNamespace = origGetHardCodedMetricsByNamespace + }) + haveBeenCalled := false + usedNamespace := "" + services.GetHardCodedMetricsByNamespace = func(namespace string) ([]models.Metric, error) { + haveBeenCalled = true + usedNamespace = namespace + return []models.Metric{}, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/metrics?region=us-east-2&namespace=AWS/DMS", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, nil)) + handler.ServeHTTP(rr, req) + res := []models.Metric{} + err := json.Unmarshal(rr.Body.Bytes(), &res) + require.Nil(t, err) + assert.True(t, haveBeenCalled) + assert.Equal(t, "AWS/DMS", usedNamespace) + }) + + t.Run("returns 500 if GetMetricsByNamespace returns an error", func(t *testing.T) { + mockListMetricsService := mocks.ListMetricsServiceMock{} + mockListMetricsService.On("GetMetricsByNamespace").Return([]models.Metric{}, fmt.Errorf("some error")) + newListMetricsService = func(pluginCtx backend.PluginContext, clientFactory models.ClientsFactoryFunc, region string) (models.ListMetricsProvider, error) { + return &mockListMetricsService, nil + } + rr := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/metrics?region=us-east-2&namespace=customNamespace", nil) + handler := http.HandlerFunc(ResourceRequestMiddleware(MetricsHandler, nil)) + handler.ServeHTTP(rr, req) + assert.Equal(t, http.StatusInternalServerError, rr.Code) + assert.Equal(t, `{"Message":"error in MetricsHandler: some error","Error":"some error","StatusCode":500}`, rr.Body.String()) + }) +} diff --git a/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go b/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go new file mode 100644 index 00000000000..a606760fad9 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/hardcoded_metrics.go @@ -0,0 +1,43 @@ +package services + +import ( + "fmt" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" +) + +var GetHardCodedDimensionKeysByNamespace = func(namespace string) ([]string, error) { + var dimensionKeys []string + exists := false + if dimensionKeys, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { + return nil, fmt.Errorf("unable to find dimensions for namespace '%q'", namespace) + } + return dimensionKeys, nil +} + +var GetHardCodedMetricsByNamespace = func(namespace string) ([]models.Metric, error) { + response := []models.Metric{} + exists := false + var metrics []string + if metrics, exists = constants.NamespaceMetricsMap[namespace]; !exists { + return nil, fmt.Errorf("unable to find metrics for namespace '%q'", namespace) + } + + for _, metric := range metrics { + response = append(response, models.Metric{Namespace: namespace, Name: metric}) + } + + return response, nil +} + +var GetAllHardCodedMetrics = func() []models.Metric { + response := []models.Metric{} + for namespace, metrics := range constants.NamespaceMetricsMap { + for _, metric := range metrics { + response = append(response, models.Metric{Namespace: namespace, Name: metric}) + } + } + + return response +} diff --git a/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go b/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go new file mode 100644 index 00000000000..ca821a18b85 --- /dev/null +++ b/pkg/tsdb/cloudwatch/services/hardcoded_metrics_test.go @@ -0,0 +1,39 @@ +package services + +import ( + "testing" + + "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHardcodedMetrics_GetHardCodedDimensionKeysByNamespace(t *testing.T) { + t.Run("Should return an error in case namespace doesnt exist in map", func(t *testing.T) { + resp, err := GetHardCodedDimensionKeysByNamespace("unknownNamespace") + require.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "unable to find dimensions for namespace '\"unknownNamespace\"'") + }) + + t.Run("Should return keys if namespace exist", func(t *testing.T) { + resp, err := GetHardCodedDimensionKeysByNamespace("AWS/EC2") + require.NoError(t, err) + assert.Equal(t, []string{"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"}, resp) + }) +} + +func TestHardcodedMetrics_GetHardCodedMetricsByNamespace(t *testing.T) { + t.Run("Should return an error in case namespace doesnt exist in map", func(t *testing.T) { + resp, err := GetHardCodedMetricsByNamespace("unknownNamespace") + require.Error(t, err) + assert.Nil(t, resp) + assert.Equal(t, err.Error(), "unable to find metrics for namespace '\"unknownNamespace\"'") + }) + + t.Run("Should return metrics if namespace exist", func(t *testing.T) { + resp, err := GetHardCodedMetricsByNamespace("AWS/IoTAnalytics") + require.NoError(t, err) + assert.Equal(t, []models.Metric{{Name: "ActionExecution", Namespace: "AWS/IoTAnalytics"}, {Name: "ActivityExecutionError", Namespace: "AWS/IoTAnalytics"}, {Name: "IncomingMessages", Namespace: "AWS/IoTAnalytics"}}, resp) + }) +} diff --git a/pkg/tsdb/cloudwatch/services/list_metrics.go b/pkg/tsdb/cloudwatch/services/list_metrics.go index 8c18588f1e9..224890743dd 100644 --- a/pkg/tsdb/cloudwatch/services/list_metrics.go +++ b/pkg/tsdb/cloudwatch/services/list_metrics.go @@ -6,7 +6,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/request" ) @@ -19,15 +18,6 @@ func NewListMetricsService(metricsClient models.MetricsClientProvider) models.Li return &ListMetricsService{metricsClient} } -func (*ListMetricsService) GetHardCodedDimensionKeysByNamespace(namespace string) ([]string, error) { - var dimensionKeys []string - exists := false - if dimensionKeys, exists = constants.NamespaceDimensionKeysMap[namespace]; !exists { - return nil, fmt.Errorf("unable to find dimensions for namespace '%q'", namespace) - } - return dimensionKeys, nil -} - func (l *ListMetricsService) GetDimensionKeysByDimensionFilter(r *request.DimensionKeysRequest) ([]string, error) { input := &cloudwatch.ListMetricsInput{} if r.Namespace != "" { @@ -126,6 +116,25 @@ func (l *ListMetricsService) GetDimensionKeysByNamespace(namespace string) ([]st return dimensionKeys, nil } +func (l *ListMetricsService) GetMetricsByNamespace(namespace string) ([]models.Metric, error) { + metrics, err := l.ListMetricsWithPageLimit(&cloudwatch.ListMetricsInput{Namespace: aws.String(namespace)}) + if err != nil { + return nil, err + } + + response := []models.Metric{} + dupCheck := make(map[string]struct{}) + for _, metric := range metrics { + if _, exists := dupCheck[*metric.MetricName]; exists { + continue + } + dupCheck[*metric.MetricName] = struct{}{} + response = append(response, models.Metric{Name: *metric.MetricName, Namespace: *metric.Namespace}) + } + + return response, nil +} + func setDimensionFilter(input *cloudwatch.ListMetricsInput, dimensionFilter []*request.Dimension) { for _, dimension := range dimensionFilter { df := &cloudwatch.DimensionFilter{ diff --git a/pkg/tsdb/cloudwatch/services/list_metrics_test.go b/pkg/tsdb/cloudwatch/services/list_metrics_test.go index b5b8057b1dc..f2dc2db0c60 100644 --- a/pkg/tsdb/cloudwatch/services/list_metrics_test.go +++ b/pkg/tsdb/cloudwatch/services/list_metrics_test.go @@ -41,23 +41,6 @@ var metricResponse = []*cloudwatch.Metric{ }, } -func TestListMetricsService_GetHardCodedDimensionKeysByNamespace(t *testing.T) { - t.Run("Should return an error in case namespace doesnt exist in map", func(t *testing.T) { - listMetricsService := NewListMetricsService(&mocks.FakeMetricsClient{}) - resp, err := listMetricsService.GetHardCodedDimensionKeysByNamespace("unknownNamespace") - require.Error(t, err) - assert.Nil(t, resp) - assert.Equal(t, err.Error(), "unable to find dimensions for namespace '\"unknownNamespace\"'") - }) - - t.Run("Should return keys if namespace exist", func(t *testing.T) { - listMetricsService := NewListMetricsService(&mocks.FakeMetricsClient{}) - resp, err := listMetricsService.GetHardCodedDimensionKeysByNamespace("AWS/EC2") - require.NoError(t, err) - assert.Equal(t, []string{"AutoScalingGroupName", "ImageId", "InstanceId", "InstanceType"}, resp) - }) -} - func TestListMetricsService_GetDimensionKeysByDimensionFilter(t *testing.T) { t.Run("Should filter out duplicates and keys matching dimension filter keys", func(t *testing.T) { fakeMetricsClient := &mocks.FakeMetricsClient{} diff --git a/public/app/plugins/datasource/cloudwatch/api.test.ts b/public/app/plugins/datasource/cloudwatch/api.test.ts index 479100a2e68..1088ddee9c4 100644 --- a/public/app/plugins/datasource/cloudwatch/api.test.ts +++ b/public/app/plugins/datasource/cloudwatch/api.test.ts @@ -59,6 +59,7 @@ describe('api', () => { it('should not initiate new api request in case a previous request had same args', async () => { const getMock = jest.fn(); const { api, resourceRequestMock } = setupMockedAPI({ getMock }); + resourceRequestMock.mockResolvedValue([]); await Promise.all([ api.getMetrics('AWS/EC2', 'us-east-1'), api.getMetrics('AWS/EC2', 'us-east-1'), diff --git a/public/app/plugins/datasource/cloudwatch/api.ts b/public/app/plugins/datasource/cloudwatch/api.ts index ba647c986fa..77ce069f40e 100644 --- a/public/app/plugins/datasource/cloudwatch/api.ts +++ b/public/app/plugins/datasource/cloudwatch/api.ts @@ -10,6 +10,7 @@ import { DescribeLogGroupsRequest, GetDimensionKeysRequest, GetDimensionValuesRequest, + MetricResponse, MultiFilters, } from './types'; @@ -56,23 +57,23 @@ export class CloudWatchAPI extends CloudWatchRequest { }); } - async getMetrics(namespace: string | undefined, region?: string) { + async getMetrics(namespace: string | undefined, region?: string): Promise>> { if (!namespace) { return []; } - return this.memoizedGetRequest('metrics', { + return this.memoizedGetRequest('metrics', { region: this.templateSrv.replace(this.getActualRegion(region)), namespace: this.templateSrv.replace(namespace), - }); + }).then((metrics) => metrics.map((m) => ({ label: m.name, value: m.name }))); } async getAllMetrics(region: string): Promise> { - const values = await this.memoizedGetRequest('all-metrics', { + const values = await this.memoizedGetRequest('all-metrics', { region: this.templateSrv.replace(this.getActualRegion(region)), }); - return values.map((v) => ({ metricName: v.value, namespace: v.text })); + return values.map((v) => ({ metricName: v.name, namespace: v.namespace })); } async getDimensionKeys({ diff --git a/public/app/plugins/datasource/cloudwatch/datasource.test.ts b/public/app/plugins/datasource/cloudwatch/datasource.test.ts index 5ca3c56a6a3..686721fd5ac 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.test.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.test.ts @@ -206,12 +206,12 @@ describe('datasource', () => { const datasource = setupMockedDataSource({ getMock: jest.fn().mockResolvedValue([ { - text: 'AWS/EC2', - value: 'CPUUtilization', + namespace: 'AWS/EC2', + name: 'CPUUtilization', }, { - text: 'AWS/Redshift', - value: 'CPUPercentage', + namespace: 'AWS/Redshift', + name: 'CPUPercentage', }, ]), }).datasource; diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index 76ef27024cc..532c94c413b 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -467,3 +467,8 @@ export interface GetDimensionValuesRequest extends ResourceRequest { metricName?: string; dimensionFilters?: Dimensions; } + +export interface MetricResponse { + name: string; + namespace: string; +}