Instrumentation: Add histogram for request duration on gRPC client to Ingesters (#3409)

* Instrumentation: Add histogram for request duration on gRPC client to Ingesters

Signed-off-by: gotjosh <josue@grafana.com>

* fix lint

Co-authored-by: Edward Welch <edward.welch@grafana.com>

pkg/ingester/client/client.go

@@ -9,13 +9,23 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
 	"github.com/opentracing/opentracing-go"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/weaveworks/common/middleware"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
 
+	cortex_middleware "github.com/cortexproject/cortex/pkg/util/middleware"
+
 	"github.com/grafana/loki/pkg/logproto"
 )
 
+var ingesterClientRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Name:    "loki_ingester_client_request_duration_seconds",
+	Help:    "Time spent doing Ingester requests.",
+	Buckets: prometheus.ExponentialBuckets(0.001, 4, 6),
+}, []string{"operation", "status_code"})
+
 type HealthAndIngesterClient interface {
 	logproto.IngesterClient
 	grpc_health_v1.HealthClient
@@ -76,8 +86,10 @@ func instrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClientInterc
 	return []grpc.UnaryClientInterceptor{
 			otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
 			middleware.ClientUserHeaderInterceptor,
+			cortex_middleware.PrometheusGRPCUnaryInstrumentation(ingesterClientRequestDuration),
 		}, []grpc.StreamClientInterceptor{
 			otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer()),
 			middleware.StreamClientUserHeaderInterceptor,
+			cortex_middleware.PrometheusGRPCStreamInstrumentation(ingesterClientRequestDuration),
 		}
 }