Fix panic on failed snapshot replay and don't hard fail replay on disabled exemplars (#9438)

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

tsdb/head.go

@@ -176,6 +176,10 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOpti
 		stats = NewHeadStats()
 	}
 
+	if !opts.EnableExemplarStorage {
+		opts.MaxExemplars.Store(0)
+	}
+
 	h := &Head{
 		wal:    wal,
 		logger: l,
@@ -211,7 +215,16 @@ func NewHead(r prometheus.Registerer, l log.Logger, wal *wal.WAL, opts *HeadOpti
 
 func (h *Head) resetInMemoryState() error {
 	var err error
-	em := NewExemplarMetrics(h.reg)
+	var em *ExemplarMetrics
+	if h.exemplars != nil {
+		ce, ok := h.exemplars.(*CircularExemplarStorage)
+		if ok {
+			em = ce.metrics
+		}
+	}
+	if em == nil {
+		em = NewExemplarMetrics(h.reg)
+	}
 	es, err := NewCircularExemplarStorage(h.opts.MaxExemplars.Load(), em)
 	if err != nil {
 		return err

tsdb/head_test.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
@@ -2753,7 +2754,16 @@ func TestChunkSnapshot(t *testing.T) {
 
 		// Test the replay of snapshot.
 		head.opts.EnableMemorySnapshotOnShutdown = true // Enabled to read from snapshot.
+
+		// Disabling exemplars to check that it does not hard fail replay
+		// https://github.com/prometheus/prometheus/issues/9437#issuecomment-933285870.
+		head.opts.EnableExemplarStorage = false
+		head.opts.MaxExemplars.Store(0)
+		expExemplars = expExemplars[:0]
+
 		openHeadAndCheckReplay()
+
+		require.Equal(t, 0.0, prom_testutil.ToFloat64(head.metrics.snapshotReplayErrorTotal))
 	}
 
 }
@@ -2805,7 +2815,8 @@ func TestSnapshotError(t *testing.T) {
 	// Create new Head which should replay this snapshot.
 	w, err := wal.NewSize(nil, nil, head.wal.Dir(), 32768, false)
 	require.NoError(t, err)
-	head, err = NewHead(nil, nil, w, head.opts, nil)
+	// Testing https://github.com/prometheus/prometheus/issues/9437 with the registry.
+	head, err = NewHead(prometheus.NewRegistry(), nil, w, head.opts, nil)
 	require.NoError(t, err)
 	require.NoError(t, head.Init(math.MinInt64))
 

tsdb/head_wal.go

@@ -948,6 +948,11 @@ Outer:
 				}
 			}
 
+			if !h.opts.EnableExemplarStorage || h.opts.MaxExemplars.Load() <= 0 {
+				// Exemplar storage is disabled.
+				continue Outer
+			}
+
 			decbuf := encoding.Decbuf{B: rec[1:]}
 
 			exemplarBuf = exemplarBuf[:0]
@@ -969,7 +974,7 @@ Outer:
 					Value:  e.V,
 					Ts:     e.T,
 				}); err != nil {
-					loopErr = errors.Wrap(err, "append exemplar")
+					loopErr = errors.Wrap(err, "add exemplar")
 					break Outer
 				}
 			}