Ruler: Adding group_key label to recording_rules_samples_queue_capacity metric (#4080)

* Adding group_key label to recording_rules_samples_queue_capacity metric

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Appeasing the linter

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

pkg/ruler/appender.go

@@ -168,7 +168,7 @@ func (a *RemoteWriteAppender) WithQueueCapacity(capacity int) error {
 		return err
 	}
 
-	a.metrics.samplesQueueCapacity.WithLabelValues(a.userID).Set(float64(capacity))
+	a.metrics.samplesQueueCapacity.WithLabelValues(a.userID, a.groupKey).Set(float64(capacity))
 	return nil
 }
 

pkg/ruler/remote_write.go

@@ -133,11 +133,12 @@ func newRemoteWriteMetrics(r prometheus.Registerer) *remoteWriteMetrics {
 			Name:      "recording_rules_samples_queued_current",
 			Help:      "Number of samples queued to be remote-written.",
 		}, []string{"tenant", "group_key"}),
+		// even though capacity can only be set per tenant, it's convenient to have a metric per rulegroup for calculations
 		samplesQueueCapacity: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: "loki",
 			Name:      "recording_rules_samples_queue_capacity",
 			Help:      "Number of samples that can be queued before eviction of oldest samples occurs.",
-		}, []string{"tenant"}),
+		}, []string{"tenant", "group_key"}),
 		remoteWriteErrors: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
 			Namespace: "loki",
 			Name:      "recording_rules_remote_write_errors",