metrics: Add `user` label to `worker_inflight_queries` (#5485)

* metrics: Add `user` label to `loki_querier_worker_inflight_queries`

* Add also `querier` label to it.

* Remove `querier`

pkg/querier/worker/metrics.go

@@ -7,7 +7,7 @@ import (
 
 type Metrics struct {
 	concurrentWorkers             prometheus.Gauge
-	inflightRequests              prometheus.Gauge
+	inflightRequests              *prometheus.GaugeVec
 	frontendClientRequestDuration *prometheus.HistogramVec
 	frontendClientsGauge          prometheus.Gauge
 }
@@ -18,10 +18,10 @@ func NewMetrics(conf Config, r prometheus.Registerer) *Metrics {
 			Name: "loki_querier_worker_concurrency",
 			Help: "Number of concurrent querier workers",
 		}),
-		inflightRequests: promauto.With(r).NewGauge(prometheus.GaugeOpts{
+		inflightRequests: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
 			Name: "loki_querier_worker_inflight_queries",
 			Help: "Number of queries being processed by the querier workers",
-		}),
+		}, []string{"user"}),
 		frontendClientRequestDuration: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
 			Name:    "loki_querier_query_frontend_request_duration_seconds",
 			Help:    "Time spend doing requests to frontend.",

pkg/querier/worker/scheduler_processor.go

@@ -130,12 +130,12 @@ func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_Quer
 
 			start := time.Now()
 			tenant, _ := tenant.TenantID(ctx)
-			sp.metrics.inflightRequests.Inc()
+			sp.metrics.inflightRequests.WithLabelValues("user", request.UserID).Inc()
 			level.Debug(logger).Log("msg", "tracking inflight request", "tenant", tenant, "op", "enqueue")
 
 			sp.runRequest(ctx, logger, request.QueryID, request.FrontendAddress, request.StatsEnabled, request.HttpRequest)
 
-			sp.metrics.inflightRequests.Dec()
+			sp.metrics.inflightRequests.WithLabelValues("user", request.UserID).Dec()
 			level.Debug(logger).Log("msg", "tracking inflight request", "tenant", tenant, "op", "dequeue", "duration", time.Since(start))
 
 			// Report back to scheduler that processing of the query has finished.