From 144786e67db4fc4998a4f42feb18beab0ff4cbc4 Mon Sep 17 00:00:00 2001 From: gotjosh Date: Mon, 1 Mar 2021 20:57:39 +0000 Subject: [PATCH] Instrumentation: Add histogram for request duration on gRPC client to Ingesters (#3409) * Instrumentation: Add histogram for request duration on gRPC client to Ingesters Signed-off-by: gotjosh * fix lint Co-authored-by: Edward Welch --- pkg/ingester/client/client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index 0f6259893a..9ff7d3ffb6 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -9,13 +9,23 @@ import ( "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/middleware" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" + cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware" + "github.com/grafana/loki/pkg/logproto" ) +var ingesterClientRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "loki_ingester_client_request_duration_seconds", + Help: "Time spent doing Ingester requests.", + Buckets: prometheus.ExponentialBuckets(0.001, 4, 6), +}, []string{"operation", "status_code"}) + type HealthAndIngesterClient interface { logproto.IngesterClient grpc_health_v1.HealthClient @@ -76,8 +86,10 @@ func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterc return []grpc.UnaryClientInterceptor{ otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()), middleware.ClientUserHeaderInterceptor, + cortex_middleware.PrometheusGRPCUnaryInstrumentation(ingesterClientRequestDuration), }, []grpc.StreamClientInterceptor{ otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()), middleware.StreamClientUserHeaderInterceptor, + cortex_middleware.PrometheusGRPCStreamInstrumentation(ingesterClientRequestDuration), } }