fix mutability bug with labels during tsdb flushing (#6116)

pkg/ingester/flush.go

@@ -354,6 +354,8 @@ func (i *Ingester) flushChunks(ctx context.Context, fp model.Fingerprint, labelP
 		return err
 	}
 
+	// NB(owen-d): No longer needed in TSDB (and is removed in that code path)
+	// It's required by historical index stores so we keep it for now.
 	labelsBuilder := labels.NewBuilder(labelPairs)
 	labelsBuilder.Set(nameLabel, logsValue)
 	metric := labelsBuilder.Labels()

pkg/storage/stores/tsdb/head_manager.go

@@ -144,12 +144,11 @@ func (m *HeadManager) Stop() error {
 
 func (m *HeadManager) Append(userID string, ls labels.Labels, chks index.ChunkMetas) error {
 	// TSDB doesnt need the __name__="log" convention the old chunk store index used.
-	for i, l := range ls {
-		if l.Name == labels.MetricName {
-			ls = append(ls[:i], ls[i+1:]...)
-			break
-		}
-	}
+	// We must create a copy of the labels here to avoid mutating the existing
+	// labels when writing across index buckets.
+	b := labels.NewBuilder(ls)
+	b.Del(labels.MetricName)
+	ls = b.Labels()
 
 	m.mtx.RLock()
 	now := time.Now()