WAL replay discard metrics (#4212)

* adds replay discard metrics

* lint

pkg/ingester/metrics.go

@@ -3,6 +3,8 @@ package ingester
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+
+	"github.com/grafana/loki/pkg/validation"
 )
 
 type ingesterMetrics struct {
@@ -13,12 +15,14 @@ type ingesterMetrics struct {
 	checkpointDuration         prometheus.Summary
 	checkpointLoggedBytesTotal prometheus.Counter
 
-	walDiskFullFailures prometheus.Counter
-	walReplayActive     prometheus.Gauge
-	walReplayDuration   prometheus.Gauge
-	walCorruptionsTotal *prometheus.CounterVec
-	walLoggedBytesTotal prometheus.Counter
-	walRecordsLogged    prometheus.Counter
+	walDiskFullFailures     prometheus.Counter
+	walReplayActive         prometheus.Gauge
+	walReplayDuration       prometheus.Gauge
+	walReplaySamplesDropped *prometheus.CounterVec
+	walReplayBytesDropped   *prometheus.CounterVec
+	walCorruptionsTotal     *prometheus.CounterVec
+	walLoggedBytesTotal     prometheus.Counter
+	walRecordsLogged        prometheus.Counter
 
 	recoveredStreamsTotal prometheus.Counter
 	recoveredChunksTotal  prometheus.Counter
@@ -45,6 +49,8 @@ func (m *ingesterMetrics) setRecoveryBytesInUse(v int64) {
 const (
 	walTypeCheckpoint = "checkpoint"
 	walTypeSegment    = "segment"
+
+	duplicateReason = "duplicate"
 )
 
 func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics {
@@ -61,6 +67,14 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics {
 			Name: "loki_ingester_wal_replay_duration_seconds",
 			Help: "Time taken to replay the checkpoint and the WAL.",
 		}),
+		walReplaySamplesDropped: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "loki_ingester_wal_discarded_samples_total",
+			Help: "WAL segment entries discarded during replay",
+		}, []string{validation.ReasonLabel}),
+		walReplayBytesDropped: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
+			Name: "loki_ingester_wal_discarded_bytes_total",
+			Help: "WAL segment bytes discarded during replay",
+		}, []string{validation.ReasonLabel}),
 		walCorruptionsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
 			Name: "loki_ingester_wal_corruptions_total",
 			Help: "Total number of WAL corruptions encountered.",

pkg/ingester/stream.go

@@ -184,6 +184,13 @@ func (s *stream) Push(
 	defer s.chunkMtx.Unlock()
 
 	if counter > 0 && counter <= s.entryCt {
+		var byteCt int
+		for _, e := range entries {
+			byteCt += len(e.Line)
+		}
+
+		s.metrics.walReplaySamplesDropped.WithLabelValues(duplicateReason).Add(float64(len(entries)))
+		s.metrics.walReplayBytesDropped.WithLabelValues(duplicateReason).Add(float64(byteCt))
 		return 0, ErrEntriesExist
 	}
 

pkg/validation/validate.go

@@ -5,7 +5,7 @@ import (
 )
 
 const (
-	reasonLabel = "reason"
+	ReasonLabel = "reason"
 	// InvalidLabels is a reason for discarding log lines which have labels that cannot be parsed.
 	InvalidLabels = "invalid_labels"
 	MissingLabels = "missing_labels"
@@ -54,7 +54,7 @@ var MutatedSamples = prometheus.NewCounterVec(
 		Name:      "mutated_samples_total",
 		Help:      "The total number of samples that have been mutated.",
 	},
-	[]string{reasonLabel, "truncated"},
+	[]string{ReasonLabel, "truncated"},
 )
 
 // MutatedBytes is a metric of the total mutated bytes, by reason.
@@ -64,7 +64,7 @@ var MutatedBytes = prometheus.NewCounterVec(
 		Name:      "mutated_bytes_total",
 		Help:      "The total number of bytes that have been mutated.",
 	},
-	[]string{reasonLabel, "truncated"},
+	[]string{ReasonLabel, "truncated"},
 )
 
 // DiscardedBytes is a metric of the total discarded bytes, by reason.
@@ -74,7 +74,7 @@ var DiscardedBytes = prometheus.NewCounterVec(
 		Name:      "discarded_bytes_total",
 		Help:      "The total number of bytes that were discarded.",
 	},
-	[]string{reasonLabel, "tenant"},
+	[]string{ReasonLabel, "tenant"},
 )
 
 // DiscardedSamples is a metric of the number of discarded samples, by reason.
@@ -84,7 +84,7 @@ var DiscardedSamples = prometheus.NewCounterVec(
 		Name:      "discarded_samples_total",
 		Help:      "The total number of samples that were discarded.",
 	},
-	[]string{reasonLabel, "tenant"},
+	[]string{ReasonLabel, "tenant"},
 )
 
 func init() {