fix(metric_aggregation): Fix duplicate metrics registration (#15142)

pkg/pattern/aggregation/metrics.go

@@ -1,12 +1,19 @@
 package aggregation
 
 import (
+	"sync"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/grafana/loki/v3/pkg/util/constants"
 )
 
+var (
+	aggMetrics  *Metrics
+	metricsOnce sync.Once
+)
+
 type Metrics struct {
 	reg prometheus.Registerer
 
@@ -28,10 +35,8 @@ type Metrics struct {
 }
 
 func NewMetrics(r prometheus.Registerer) *Metrics {
-	var m Metrics
-	m.reg = r
-
-	m = Metrics{
+	metricsOnce.Do(func() {
+		aggMetrics = &Metrics{
 			chunks: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
 				Namespace: constants.Loki,
 				Subsystem: "pattern_ingester",
@@ -103,21 +108,7 @@ func NewMetrics(r prometheus.Registerer) *Metrics {
 				Help:      "Total number of write timeouts.",
 			}, []string{"tenant_id"}),
 		}
+	})
 
-	if m.reg != nil {
-		m.reg.MustRegister(
-			m.chunks,
-			m.samples,
-			m.pushErrors,
-			m.pushRetries,
-			m.pushSuccesses,
-			m.payloadSize,
-			m.streamsPerPush,
-			m.entriesPerPush,
-			m.servicesTracked,
-			m.writeTimeout,
-		)
-	}
-
-	return &m
+	return aggMetrics
 }

pkg/pattern/aggregation/push.go

@@ -17,7 +17,6 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/golang/snappy"
 	"github.com/opentracing/opentracing-go"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
@@ -112,7 +111,7 @@ func NewPush(
 	useTLS bool,
 	backoffCfg *backoff.Config,
 	logger log.Logger,
-	registrer prometheus.Registerer,
+	metrics *Metrics,
 ) (*Push, error) {
 	client, err := config.NewClientFromConfig(cfg, "pattern-ingester-push", config.WithHTTP2Disabled())
 	if err != nil {
@@ -147,7 +146,7 @@ func NewPush(
 		entries: entries{
 			entries: make([]entry, 0),
 		},
-		metrics: NewMetrics(registrer),
+		metrics: metrics,
 	}
 
 	go p.run(pushPeriod)
@@ -284,12 +283,12 @@ func (p *Push) run(pushPeriod time.Duration) {
 				}
 
 				if !backoff.Ongoing() {
-					level.Error(p.logger).Log("msg", "failed to send entry, retries exhausted, entry will be dropped", "entry", "status", status, "error", err)
+					level.Error(p.logger).Log("msg", "failed to send entry, retries exhausted, entry will be dropped", "status", status, "error", err)
 					pushTicker.Reset(pushPeriod)
 					break
 				}
 				level.Warn(p.logger).
-					Log("msg", "failed to send entry, retrying", "entry", "status", status, "error", err)
+					Log("msg", "failed to send entry, retrying", "status", status, "error", err)
 				backoff.Wait()
 			}
 

pkg/pattern/aggregation/push_test.go

@@ -58,7 +58,7 @@ func Test_Push(t *testing.T) {
 			false,
 			&backoff,
 			log.NewNopLogger(),
-			nil,
+			NewMetrics(nil),
 		)
 		require.NoError(t, err)
 		ts, payload := testPayload()
@@ -83,7 +83,7 @@ func Test_Push(t *testing.T) {
 			"user", "secret",
 			false,
 			&backoff,
-			log.NewNopLogger(), nil,
+			log.NewNopLogger(), NewMetrics(nil),
 		)
 		require.NoError(t, err)
 		ts, payload := testPayload()

pkg/pattern/ingester.go

@@ -392,6 +392,7 @@ func (i *Ingester) GetOrCreateInstance(instanceID string) (*instance, error) { /
 
 		aggCfg := i.cfg.MetricAggregation
 		if i.limits.MetricAggregationEnabled(instanceID) {
+			metricAggregationMetrics := aggregation.NewMetrics(i.registerer)
 			writer, err = aggregation.NewPush(
 				aggCfg.LokiAddr,
 				instanceID,
@@ -403,7 +404,7 @@ func (i *Ingester) GetOrCreateInstance(instanceID string) (*instance, error) { /
 				aggCfg.UseTLS,
 				&aggCfg.BackoffConfig,
 				i.logger,
-				i.registerer,
+				metricAggregationMetrics,
 			)
 			if err != nil {
 				return nil, err