Resolve conflicts between multiple exported label prefixes (#9479)

Resolve conflicts between multiple exported label prefixes

Signed-off-by: Shirley Leu <shirley.w.leu@gmail.com>

scrape/scrape.go

@@ -24,6 +24,7 @@ import (
 	"math"
 	"net/http"
 	"reflect"
+	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -648,15 +649,19 @@ func mutateSampleLabels(lset labels.Labels, target *Target, honor bool, rc []*re
 			}
 		}
 	} else {
+		var conflictingExposedLabels labels.Labels
 		for _, l := range target.Labels() {
-			// existingValue will be empty if l.Name doesn't exist.
 			existingValue := lset.Get(l.Name)
 			if existingValue != "" {
-				lb.Set(model.ExportedLabelPrefix+l.Name, existingValue)
+				conflictingExposedLabels = append(conflictingExposedLabels, labels.Label{Name: l.Name, Value: existingValue})
 			}
 			// It is now safe to set the target label.
 			lb.Set(l.Name, l.Value)
 		}
+
+		if len(conflictingExposedLabels) > 0 {
+			resolveConflictingExposedLabels(lb, conflictingExposedLabels)
+		}
 	}
 
 	res := lb.Labels()
@@ -668,6 +673,38 @@ func mutateSampleLabels(lset labels.Labels, target *Target, honor bool, rc []*re
 	return res
 }
 
+func resolveConflictingExposedLabels(lb *labels.Builder, conflictingExposedLabels labels.Labels) {
+	sort.SliceStable(conflictingExposedLabels, func(i, j int) bool {
+		return len(conflictingExposedLabels[i].Name) < len(conflictingExposedLabels[j].Name)
+	})
+
+	allLabelNames := map[string]struct{}{}
+	for _, v := range lb.Labels() {
+		allLabelNames[v.Name] = struct{}{}
+	}
+
+	resolved := createNewLabels(allLabelNames, conflictingExposedLabels, nil)
+	for _, l := range resolved {
+		lb.Set(l.Name, l.Value)
+	}
+}
+
+func createNewLabels(existingNames map[string]struct{}, conflictingLabels, resolvedLabels labels.Labels) labels.Labels {
+	for i := 0; i < len(conflictingLabels); i++ {
+		newName := model.ExportedLabelPrefix + conflictingLabels[i].Name
+		if _, ok := existingNames[newName]; !ok {
+			resolvedLabels = append(resolvedLabels, labels.Label{Name: newName, Value: conflictingLabels[i].Value})
+			conflictingLabels = append(conflictingLabels[:i], conflictingLabels[i+1:]...)
+			i--
+			existingNames[newName] = struct{}{}
+		} else {
+			conflictingLabels[i] = labels.Label{Name: newName, Value: conflictingLabels[i].Value}
+			return createNewLabels(existingNames, conflictingLabels, resolvedLabels)
+		}
+	}
+	return resolvedLabels
+}
+
 func mutateReportSampleLabels(lset labels.Labels, target *Target) labels.Labels {
 	lb := labels.NewBuilder(lset)
 

scrape/scrape_test.go

@@ -1379,6 +1379,81 @@ func TestScrapeLoopAppend(t *testing.T) {
 	}
 }
 
+func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
+	testcases := map[string]struct {
+		targetLabels  []string
+		exposedLabels string
+		expected      []string
+	}{
+		"One target label collides with existing label": {
+			targetLabels:  []string{"foo", "2"},
+			exposedLabels: `metric{foo="1"} 0`,
+			expected:      []string{"__name__", "metric", "exported_foo", "1", "foo", "2"},
+		},
+
+		"One target label collides with existing label, plus target label already with prefix 'exported'": {
+			targetLabels:  []string{"foo", "2", "exported_foo", "3"},
+			exposedLabels: `metric{foo="1"} 0`,
+			expected:      []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "3", "foo", "2"},
+		},
+		"One target label collides with existing label, plus existing label already with prefix 'exported": {
+			targetLabels:  []string{"foo", "3"},
+			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			expected:      []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "2", "foo", "3"},
+		},
+		"One target label collides with existing label, both already with prefix 'exported'": {
+			targetLabels:  []string{"exported_foo", "2"},
+			exposedLabels: `metric{exported_foo="1"} 0`,
+			expected:      []string{"__name__", "metric", "exported_exported_foo", "1", "exported_foo", "2"},
+		},
+		"Two target labels collide with existing labels, both with and without prefix 'exported'": {
+			targetLabels:  []string{"foo", "3", "exported_foo", "4"},
+			exposedLabels: `metric{foo="1" exported_foo="2"} 0`,
+			expected: []string{"__name__", "metric", "exported_exported_foo", "1", "exported_exported_exported_foo",
+				"2", "exported_foo", "4", "foo", "3"},
+		},
+		"Extreme example": {
+			targetLabels:  []string{"foo", "0", "exported_exported_foo", "1", "exported_exported_exported_foo", "2"},
+			exposedLabels: `metric{foo="3" exported_foo="4" exported_exported_exported_foo="5"} 0`,
+			expected: []string{
+				"__name__", "metric",
+				"exported_exported_exported_exported_exported_foo", "5",
+				"exported_exported_exported_exported_foo", "3",
+				"exported_exported_exported_foo", "2",
+				"exported_exported_foo", "1",
+				"exported_foo", "4",
+				"foo", "0",
+			},
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			app := &collectResultAppender{}
+			sl := newScrapeLoop(context.Background(), nil, nil, nil,
+				func(l labels.Labels) labels.Labels {
+					return mutateSampleLabels(l, &Target{labels: labels.FromStrings(tc.targetLabels...)}, false, nil)
+				},
+				nil,
+				func(ctx context.Context) storage.Appender { return app }, nil, 0, true, 0, nil, 0, 0, false,
+			)
+			slApp := sl.appender(context.Background())
+			_, _, _, err := sl.append(slApp, []byte(tc.exposedLabels), "", time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC))
+			require.NoError(t, err)
+
+			require.NoError(t, slApp.Commit())
+
+			require.Equal(t, []sample{
+				{
+					metric: labels.FromStrings(tc.expected...),
+					t:      timestamp.FromTime(time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC)),
+					v:      0,
+				},
+			}, app.result)
+		})
+	}
+}
+
 func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) {
 	// collectResultAppender's AddFast always returns ErrNotFound if we don't give it a next.
 	app := &collectResultAppender{}