mirror of https://github.com/grafana/grafana
Plugins: Refactoring: Implement plugin instrumentation as a middleware (#76011)
* Plugins: Refactor instrumentation as plugin client middleware * Simplify repeated code * Fix compilation error * Add comments * Moved status and endpoint consts to utils.go * Fix wrong endpoint name in CheckHealth InstrumentationMiddleware * Add tests * Fix wrong endpoint value in instrumentPluginRequestSize * removed todo * PR review feedback: use MustRegister * PR review feedback: move tracing middleware before instrumentation middleware * PR review feedback: removed decommissioned check * PR review feedback: extract prometheus metrics into separate variablespull/75748/head
Giuseppe Guerra
2 years ago
committed by
GitHub
parent
57b99728ad
commit
cfcfbe4aaa
@ -1,136 +0,0 @@ |
||||
// Package instrumentation contains backend plugin instrumentation logic.
|
||||
package instrumentation |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/client_golang/prometheus/promauto" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/infra/tracing" |
||||
"github.com/grafana/grafana/pkg/plugins/backendplugin" |
||||
) |
||||
|
||||
var ( |
||||
pluginRequestCounter = promauto.NewCounterVec(prometheus.CounterOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_total", |
||||
Help: "The total amount of plugin requests", |
||||
}, []string{"plugin_id", "endpoint", "status", "target"}) |
||||
|
||||
pluginRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_duration_milliseconds", |
||||
Help: "Plugin request duration", |
||||
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}, |
||||
}, []string{"plugin_id", "endpoint", "target"}) |
||||
|
||||
pluginRequestSizeHistogram = promauto.NewHistogramVec( |
||||
prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_size_bytes", |
||||
Help: "histogram of plugin request sizes returned", |
||||
Buckets: []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576}, |
||||
}, []string{"source", "plugin_id", "endpoint", "target"}, |
||||
) |
||||
|
||||
PluginRequestDurationSeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_duration_seconds", |
||||
Help: "Plugin request duration in seconds", |
||||
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25}, |
||||
}, []string{"source", "plugin_id", "endpoint", "status", "target"}) |
||||
) |
||||
|
||||
const ( |
||||
statusOK = "ok" |
||||
statusError = "error" |
||||
statusCancelled = "cancelled" |
||||
|
||||
endpointCallResource = "callResource" |
||||
endpointCheckHealth = "checkHealth" |
||||
endpointCollectMetrics = "collectMetrics" |
||||
endpointQueryData = "queryData" |
||||
) |
||||
|
||||
// instrumentPluginRequest instruments success rate and latency of `fn`
|
||||
func instrumentPluginRequest(ctx context.Context, cfg Cfg, pluginCtx *backend.PluginContext, endpoint string, fn func(ctx context.Context) error) error { |
||||
status := statusOK |
||||
start := time.Now() |
||||
|
||||
ctx = instrumentContext(ctx, endpoint, *pluginCtx) |
||||
err := fn(ctx) |
||||
if err != nil { |
||||
status = statusError |
||||
if errors.Is(err, context.Canceled) { |
||||
status = statusCancelled |
||||
} |
||||
} |
||||
|
||||
elapsed := time.Since(start) |
||||
|
||||
pluginRequestDurationWithLabels := pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, string(cfg.Target)) |
||||
pluginRequestCounterWithLabels := pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, string(cfg.Target)) |
||||
pluginRequestDurationSecondsWithLabels := PluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, string(cfg.Target)) |
||||
|
||||
if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" { |
||||
pluginRequestDurationWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( |
||||
float64(elapsed/time.Millisecond), prometheus.Labels{"traceID": traceID}, |
||||
) |
||||
pluginRequestCounterWithLabels.(prometheus.ExemplarAdder).AddWithExemplar(1, prometheus.Labels{"traceID": traceID}) |
||||
pluginRequestDurationSecondsWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( |
||||
elapsed.Seconds(), prometheus.Labels{"traceID": traceID}, |
||||
) |
||||
} else { |
||||
pluginRequestDurationWithLabels.Observe(float64(elapsed / time.Millisecond)) |
||||
pluginRequestCounterWithLabels.Inc() |
||||
pluginRequestDurationSecondsWithLabels.Observe(elapsed.Seconds()) |
||||
} |
||||
|
||||
return err |
||||
} |
||||
|
||||
func instrumentContext(ctx context.Context, endpoint string, pCtx backend.PluginContext) context.Context { |
||||
p := []any{"endpoint", endpoint, "pluginId", pCtx.PluginID} |
||||
if pCtx.DataSourceInstanceSettings != nil { |
||||
p = append(p, "dsName", pCtx.DataSourceInstanceSettings.Name) |
||||
p = append(p, "dsUID", pCtx.DataSourceInstanceSettings.UID) |
||||
} |
||||
if pCtx.User != nil { |
||||
p = append(p, "uname", pCtx.User.Login) |
||||
} |
||||
return log.WithContextualAttributes(ctx, p) |
||||
} |
||||
|
||||
type Cfg struct { |
||||
Target backendplugin.Target |
||||
} |
||||
|
||||
// InstrumentCollectMetrics instruments collectMetrics.
|
||||
func InstrumentCollectMetrics(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func(ctx context.Context) error) error { |
||||
return instrumentPluginRequest(ctx, cfg, req, endpointCollectMetrics, fn) |
||||
} |
||||
|
||||
// InstrumentCheckHealthRequest instruments checkHealth.
|
||||
func InstrumentCheckHealthRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, fn func(ctx context.Context) error) error { |
||||
return instrumentPluginRequest(ctx, cfg, req, endpointCheckHealth, fn) |
||||
} |
||||
|
||||
// InstrumentCallResourceRequest instruments callResource.
|
||||
func InstrumentCallResourceRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, requestSize float64, fn func(ctx context.Context) error) error { |
||||
pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointCallResource, |
||||
string(cfg.Target)).Observe(requestSize) |
||||
return instrumentPluginRequest(ctx, cfg, req, endpointCallResource, fn) |
||||
} |
||||
|
||||
// InstrumentQueryDataRequest instruments success rate and latency of query data requests.
|
||||
func InstrumentQueryDataRequest(ctx context.Context, req *backend.PluginContext, cfg Cfg, |
||||
requestSize float64, fn func(ctx context.Context) error) error { |
||||
pluginRequestSizeHistogram.WithLabelValues("grafana-backend", req.PluginID, endpointQueryData, |
||||
string(cfg.Target)).Observe(requestSize) |
||||
return instrumentPluginRequest(ctx, cfg, req, endpointQueryData, fn) |
||||
} |
||||
@ -0,0 +1,214 @@ |
||||
package clientmiddleware |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/infra/tracing" |
||||
"github.com/grafana/grafana/pkg/plugins" |
||||
"github.com/grafana/grafana/pkg/plugins/manager/registry" |
||||
) |
||||
|
||||
// pluginMetrics contains the prometheus metrics used by the InstrumentationMiddleware.
|
||||
type pluginMetrics struct { |
||||
pluginRequestCounter *prometheus.CounterVec |
||||
pluginRequestDuration *prometheus.HistogramVec |
||||
pluginRequestSize *prometheus.HistogramVec |
||||
pluginRequestDurationSeconds *prometheus.HistogramVec |
||||
} |
||||
|
||||
// InstrumentationMiddleware is a middleware that instruments plugin requests.
|
||||
// It tracks requests count, duration and size as prometheus metrics.
|
||||
// It also enriches the [context.Context] with a contextual logger containing plugin and request details.
|
||||
// For those reasons, this middleware should live at the top of the middleware stack.
|
||||
type InstrumentationMiddleware struct { |
||||
pluginMetrics |
||||
pluginRegistry registry.Service |
||||
next plugins.Client |
||||
} |
||||
|
||||
func newInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) *InstrumentationMiddleware { |
||||
pluginRequestCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_total", |
||||
Help: "The total amount of plugin requests", |
||||
}, []string{"plugin_id", "endpoint", "status", "target"}) |
||||
pluginRequestDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_duration_milliseconds", |
||||
Help: "Plugin request duration", |
||||
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25, 50, 100}, |
||||
}, []string{"plugin_id", "endpoint", "target"}) |
||||
pluginRequestSize := prometheus.NewHistogramVec( |
||||
prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_size_bytes", |
||||
Help: "histogram of plugin request sizes returned", |
||||
Buckets: []float64{128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576}, |
||||
}, []string{"source", "plugin_id", "endpoint", "target"}, |
||||
) |
||||
pluginRequestDurationSeconds := prometheus.NewHistogramVec(prometheus.HistogramOpts{ |
||||
Namespace: "grafana", |
||||
Name: "plugin_request_duration_seconds", |
||||
Help: "Plugin request duration in seconds", |
||||
Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 25}, |
||||
}, []string{"source", "plugin_id", "endpoint", "status", "target"}) |
||||
promRegisterer.MustRegister( |
||||
pluginRequestCounter, |
||||
pluginRequestDuration, |
||||
pluginRequestSize, |
||||
pluginRequestDurationSeconds, |
||||
) |
||||
return &InstrumentationMiddleware{ |
||||
pluginMetrics: pluginMetrics{ |
||||
pluginRequestCounter: pluginRequestCounter, |
||||
pluginRequestDuration: pluginRequestDuration, |
||||
pluginRequestSize: pluginRequestSize, |
||||
pluginRequestDurationSeconds: pluginRequestDurationSeconds, |
||||
}, |
||||
pluginRegistry: pluginRegistry, |
||||
} |
||||
} |
||||
|
||||
// NewInstrumentationMiddleware returns a new InstrumentationMiddleware.
|
||||
func NewInstrumentationMiddleware(promRegisterer prometheus.Registerer, pluginRegistry registry.Service) plugins.ClientMiddleware { |
||||
imw := newInstrumentationMiddleware(promRegisterer, pluginRegistry) |
||||
return plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { |
||||
imw.next = next |
||||
return imw |
||||
}) |
||||
} |
||||
|
||||
// pluginTarget returns the value for the "target" Prometheus label for the given plugin ID.
|
||||
func (m *InstrumentationMiddleware) pluginTarget(ctx context.Context, pluginID string) (string, error) { |
||||
p, exists := m.pluginRegistry.Plugin(ctx, pluginID) |
||||
if !exists { |
||||
return "", plugins.ErrPluginNotRegistered |
||||
} |
||||
return string(p.Target()), nil |
||||
} |
||||
|
||||
// instrumentContext adds a contextual logger with plugin and request details to the given context.
|
||||
func instrumentContext(ctx context.Context, endpoint string, pCtx backend.PluginContext) context.Context { |
||||
p := []any{"endpoint", endpoint, "pluginId", pCtx.PluginID} |
||||
if pCtx.DataSourceInstanceSettings != nil { |
||||
p = append(p, "dsName", pCtx.DataSourceInstanceSettings.Name) |
||||
p = append(p, "dsUID", pCtx.DataSourceInstanceSettings.UID) |
||||
} |
||||
if pCtx.User != nil { |
||||
p = append(p, "uname", pCtx.User.Login) |
||||
} |
||||
return log.WithContextualAttributes(ctx, p) |
||||
} |
||||
|
||||
// instrumentPluginRequestSize tracks the size of the given request in the m.pluginRequestSize metric.
|
||||
func (m *InstrumentationMiddleware) instrumentPluginRequestSize(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, requestSize float64) error { |
||||
target, err := m.pluginTarget(ctx, pluginCtx.PluginID) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
m.pluginRequestSize.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, target).Observe(requestSize) |
||||
return nil |
||||
} |
||||
|
||||
// instrumentPluginRequest increments the m.pluginRequestCounter metric and tracks the duration of the given request.
|
||||
func (m *InstrumentationMiddleware) instrumentPluginRequest(ctx context.Context, pluginCtx backend.PluginContext, endpoint string, fn func(context.Context) error) error { |
||||
target, err := m.pluginTarget(ctx, pluginCtx.PluginID) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
status := statusOK |
||||
start := time.Now() |
||||
|
||||
ctx = instrumentContext(ctx, endpoint, pluginCtx) |
||||
err = fn(ctx) |
||||
if err != nil { |
||||
status = statusError |
||||
if errors.Is(err, context.Canceled) { |
||||
status = statusCancelled |
||||
} |
||||
} |
||||
|
||||
elapsed := time.Since(start) |
||||
|
||||
pluginRequestDurationWithLabels := m.pluginRequestDuration.WithLabelValues(pluginCtx.PluginID, endpoint, target) |
||||
pluginRequestCounterWithLabels := m.pluginRequestCounter.WithLabelValues(pluginCtx.PluginID, endpoint, status, target) |
||||
pluginRequestDurationSecondsWithLabels := m.pluginRequestDurationSeconds.WithLabelValues("grafana-backend", pluginCtx.PluginID, endpoint, status, target) |
||||
|
||||
if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" { |
||||
pluginRequestDurationWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( |
||||
float64(elapsed/time.Millisecond), prometheus.Labels{"traceID": traceID}, |
||||
) |
||||
pluginRequestCounterWithLabels.(prometheus.ExemplarAdder).AddWithExemplar(1, prometheus.Labels{"traceID": traceID}) |
||||
pluginRequestDurationSecondsWithLabels.(prometheus.ExemplarObserver).ObserveWithExemplar( |
||||
elapsed.Seconds(), prometheus.Labels{"traceID": traceID}, |
||||
) |
||||
} else { |
||||
pluginRequestDurationWithLabels.Observe(float64(elapsed / time.Millisecond)) |
||||
pluginRequestCounterWithLabels.Inc() |
||||
pluginRequestDurationSecondsWithLabels.Observe(elapsed.Seconds()) |
||||
} |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) QueryData(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) { |
||||
var requestSize float64 |
||||
for _, v := range req.Queries { |
||||
requestSize += float64(len(v.JSON)) |
||||
} |
||||
if err := m.instrumentPluginRequestSize(ctx, req.PluginContext, endpointQueryData, requestSize); err != nil { |
||||
return nil, err |
||||
} |
||||
var resp *backend.QueryDataResponse |
||||
err := m.instrumentPluginRequest(ctx, req.PluginContext, endpointQueryData, func(ctx context.Context) (innerErr error) { |
||||
resp, innerErr = m.next.QueryData(ctx, req) |
||||
return innerErr |
||||
}) |
||||
return resp, err |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error { |
||||
if err := m.instrumentPluginRequestSize(ctx, req.PluginContext, endpointCallResource, float64(len(req.Body))); err != nil { |
||||
return err |
||||
} |
||||
return m.instrumentPluginRequest(ctx, req.PluginContext, endpointCallResource, func(ctx context.Context) error { |
||||
return m.next.CallResource(ctx, req, sender) |
||||
}) |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) CheckHealth(ctx context.Context, req *backend.CheckHealthRequest) (*backend.CheckHealthResult, error) { |
||||
var result *backend.CheckHealthResult |
||||
err := m.instrumentPluginRequest(ctx, req.PluginContext, endpointCheckHealth, func(ctx context.Context) (innerErr error) { |
||||
result, innerErr = m.next.CheckHealth(ctx, req) |
||||
return |
||||
}) |
||||
return result, err |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) CollectMetrics(ctx context.Context, req *backend.CollectMetricsRequest) (*backend.CollectMetricsResult, error) { |
||||
var result *backend.CollectMetricsResult |
||||
err := m.instrumentPluginRequest(ctx, req.PluginContext, endpointCollectMetrics, func(ctx context.Context) (innerErr error) { |
||||
result, innerErr = m.next.CollectMetrics(ctx, req) |
||||
return |
||||
}) |
||||
return result, err |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) SubscribeStream(ctx context.Context, req *backend.SubscribeStreamRequest) (*backend.SubscribeStreamResponse, error) { |
||||
return m.next.SubscribeStream(ctx, req) |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) PublishStream(ctx context.Context, req *backend.PublishStreamRequest) (*backend.PublishStreamResponse, error) { |
||||
return m.next.PublishStream(ctx, req) |
||||
} |
||||
|
||||
func (m *InstrumentationMiddleware) RunStream(ctx context.Context, req *backend.RunStreamRequest, sender *backend.StreamSender) error { |
||||
return m.next.RunStream(ctx, req, sender) |
||||
} |
||||
@ -0,0 +1,164 @@ |
||||
package clientmiddleware |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"testing" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/client_golang/prometheus/testutil" |
||||
dto "github.com/prometheus/client_model/go" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/grafana/pkg/plugins" |
||||
"github.com/grafana/grafana/pkg/plugins/backendplugin" |
||||
"github.com/grafana/grafana/pkg/plugins/manager/client/clienttest" |
||||
"github.com/grafana/grafana/pkg/plugins/manager/fakes" |
||||
) |
||||
|
||||
func TestInstrumentationMiddleware(t *testing.T) { |
||||
const ( |
||||
pluginID = "plugin-id" |
||||
|
||||
metricRequestTotal = "grafana_plugin_request_total" |
||||
metricRequestDurationMs = "grafana_plugin_request_duration_milliseconds" |
||||
metricRequestDurationS = "grafana_plugin_request_duration_seconds" |
||||
metricRequestSize = "grafana_plugin_request_size_bytes" |
||||
) |
||||
|
||||
pCtx := backend.PluginContext{PluginID: pluginID} |
||||
|
||||
t.Run("should instrument requests", func(t *testing.T) { |
||||
for _, tc := range []struct { |
||||
expEndpoint string |
||||
fn func(cdt *clienttest.ClientDecoratorTest) error |
||||
shouldInstrumentRequestSize bool |
||||
}{ |
||||
{ |
||||
expEndpoint: endpointCheckHealth, |
||||
fn: func(cdt *clienttest.ClientDecoratorTest) error { |
||||
_, err := cdt.Decorator.CheckHealth(context.Background(), &backend.CheckHealthRequest{PluginContext: pCtx}) |
||||
return err |
||||
}, |
||||
shouldInstrumentRequestSize: false, |
||||
}, |
||||
{ |
||||
expEndpoint: endpointCallResource, |
||||
fn: func(cdt *clienttest.ClientDecoratorTest) error { |
||||
return cdt.Decorator.CallResource(context.Background(), &backend.CallResourceRequest{PluginContext: pCtx}, nopCallResourceSender) |
||||
}, |
||||
shouldInstrumentRequestSize: true, |
||||
}, |
||||
{ |
||||
expEndpoint: endpointQueryData, |
||||
fn: func(cdt *clienttest.ClientDecoratorTest) error { |
||||
_, err := cdt.Decorator.QueryData(context.Background(), &backend.QueryDataRequest{PluginContext: pCtx}) |
||||
return err |
||||
}, |
||||
shouldInstrumentRequestSize: true, |
||||
}, |
||||
{ |
||||
expEndpoint: endpointCollectMetrics, |
||||
fn: func(cdt *clienttest.ClientDecoratorTest) error { |
||||
_, err := cdt.Decorator.CollectMetrics(context.Background(), &backend.CollectMetricsRequest{PluginContext: pCtx}) |
||||
return err |
||||
}, |
||||
shouldInstrumentRequestSize: false, |
||||
}, |
||||
} { |
||||
t.Run(tc.expEndpoint, func(t *testing.T) { |
||||
promRegistry := prometheus.NewRegistry() |
||||
pluginsRegistry := fakes.NewFakePluginRegistry() |
||||
require.NoError(t, pluginsRegistry.Add(context.Background(), &plugins.Plugin{ |
||||
JSONData: plugins.JSONData{ID: pluginID, Backend: true}, |
||||
})) |
||||
|
||||
mw := newInstrumentationMiddleware(promRegistry, pluginsRegistry) |
||||
cdt := clienttest.NewClientDecoratorTest(t, clienttest.WithMiddlewares( |
||||
plugins.ClientMiddlewareFunc(func(next plugins.Client) plugins.Client { |
||||
mw.next = next |
||||
return mw |
||||
}), |
||||
)) |
||||
require.NoError(t, tc.fn(cdt)) |
||||
|
||||
// Ensure the correct metrics have been incremented/observed
|
||||
require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestTotal)) |
||||
require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestDurationMs)) |
||||
require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestDurationS)) |
||||
|
||||
counter := mw.pluginMetrics.pluginRequestCounter.WithLabelValues(pluginID, tc.expEndpoint, statusOK, string(backendplugin.TargetUnknown)) |
||||
require.Equal(t, 1.0, testutil.ToFloat64(counter)) |
||||
for _, m := range []string{metricRequestDurationMs, metricRequestDurationS} { |
||||
require.NoError(t, checkHistogram(promRegistry, m, map[string]string{ |
||||
"plugin_id": pluginID, |
||||
"endpoint": tc.expEndpoint, |
||||
"target": string(backendplugin.TargetUnknown), |
||||
})) |
||||
} |
||||
if tc.shouldInstrumentRequestSize { |
||||
require.Equal(t, 1, testutil.CollectAndCount(promRegistry, metricRequestSize), "request size should have been instrumented") |
||||
require.NoError(t, checkHistogram(promRegistry, metricRequestSize, map[string]string{ |
||||
"plugin_id": pluginID, |
||||
"endpoint": tc.expEndpoint, |
||||
"target": string(backendplugin.TargetUnknown), |
||||
"source": "grafana-backend", |
||||
}), "request size should have been instrumented") |
||||
} |
||||
}) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
// checkHistogram is a utility function that checks if a histogram with the given name and label values exists
|
||||
// and has been observed at least once.
|
||||
func checkHistogram(promRegistry *prometheus.Registry, expMetricName string, expLabels map[string]string) error { |
||||
metrics, err := promRegistry.Gather() |
||||
if err != nil { |
||||
return fmt.Errorf("gather: %w", err) |
||||
} |
||||
var metricFamily *dto.MetricFamily |
||||
for _, mf := range metrics { |
||||
if *mf.Name == expMetricName { |
||||
metricFamily = mf |
||||
break |
||||
} |
||||
} |
||||
if metricFamily == nil { |
||||
return fmt.Errorf("metric %q not found", expMetricName) |
||||
} |
||||
var foundLabels int |
||||
var metric *dto.Metric |
||||
for _, m := range metricFamily.Metric { |
||||
for _, l := range m.GetLabel() { |
||||
v, ok := expLabels[*l.Name] |
||||
if !ok { |
||||
continue |
||||
} |
||||
if v != *l.Value { |
||||
return fmt.Errorf("expected label %q to have value %q, got %q", *l.Name, v, *l.Value) |
||||
} |
||||
foundLabels++ |
||||
} |
||||
if foundLabels == 0 { |
||||
continue |
||||
} |
||||
if foundLabels != len(expLabels) { |
||||
return fmt.Errorf("expected %d labels, got %d", len(expLabels), foundLabels) |
||||
} |
||||
metric = m |
||||
break |
||||
} |
||||
if metric == nil { |
||||
return fmt.Errorf("could not find metric with labels %v", expLabels) |
||||
} |
||||
if metric.Histogram == nil { |
||||
return fmt.Errorf("metric %q is not a histogram", expMetricName) |
||||
} |
||||
if metric.Histogram.SampleCount == nil || *metric.Histogram.SampleCount == 0 { |
||||
return errors.New("found metric but no samples have been collected") |
||||
} |
||||
return nil |
||||
} |
||||
Loading…
Reference in new issue