Loki: Add new sharding metrics (histogram based) (#7762)

**What this PR does / why we need it**:
Add histogram-version of two existing stream sharding metrics. The new
metrics are:
- `rate_store_stream_shards`: Will show us the distribution of shards
reported by ingesters. This will help us on monitoring the number of
shards we're dealing with. As of now we only have access to the max
number of shards seen, which is helpful but not enough.
- `rate_store_stream_rate_bytes`: Same as below: will help us understand
the distribution of stream sizes instead of only showing us the max
stream size seen.

pkg/distributor/ratestore.go

@@ -123,6 +123,7 @@ func (s *rateStore) updateAllRates(ctx context.Context) error {
 			continue
 		}
 
+		s.metrics.streamRate.Observe(float64(rate.rate))
 		maxRate = max(maxRate, rate.rate)
 	}
 
@@ -166,6 +167,7 @@ func (s *rateStore) aggregateByShard(streamRates map[string]*logproto.StreamRate
 
 	var maxShards int64
 	for _, v := range shardCount {
+		s.metrics.streamShardCount.Observe(float64(v))
 		maxShards = max(maxShards, int64(v))
 	}
 

pkg/distributor/ratestore_metrics.go

@@ -10,7 +10,9 @@ type ratestoreMetrics struct {
 	rateRefreshFailures *prometheus.CounterVec
 	streamCount         prometheus.Gauge
 	maxStreamShardCount prometheus.Gauge
+	streamShardCount    prometheus.Histogram
 	maxStreamRate       prometheus.Gauge
+	streamRate          prometheus.Histogram
 	maxUniqueStreamRate prometheus.Gauge
 	refreshDuration     *instrument.HistogramCollector
 }
@@ -32,11 +34,23 @@ func newRateStoreMetrics(reg prometheus.Registerer) *ratestoreMetrics {
 			Name:      "rate_store_max_stream_shards",
 			Help:      "The number of shards for a single stream reported by ingesters during a sync operation.",
 		}),
+		streamShardCount: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
+			Namespace: "loki",
+			Name:      "rate_store_stream_shards",
+			Help:      "The distribution of number of shards for a single stream reported by ingesters during a sync operation.",
+			Buckets:   []float64{0, 2, 4, 8, 16, 32, 64, 128},
+		}),
 		maxStreamRate: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
 			Namespace: "loki",
 			Name:      "rate_store_max_stream_rate_bytes",
 			Help:      "The maximum stream rate for any stream reported by ingesters during a sync operation. Sharded Streams are combined.",
 		}),
+		streamRate: promauto.With(reg).NewHistogram(prometheus.HistogramOpts{
+			Namespace: "loki",
+			Name:      "rate_store_stream_rate_bytes",
+			Help:      "The distribution of stream rates for any stream reported by ingesters during a sync operation. Sharded Streams are combined.",
+			Buckets:   prometheus.ExponentialBuckets(20000, 2, 14), // biggest bucket is 20000*2^(14-1) = 163,840,000 (~163.84MB)
+		}),
 		maxUniqueStreamRate: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
 			Namespace: "loki",
 			Name:      "rate_store_max_unique_stream_rate_bytes",