Merge pull request #13129 from fatsheep9146/reduce-resolution-automatically

Native Histograms: automatically reduce resolution rather than fail scrape

scrape/scrape_test.go

@@ -1624,6 +1624,29 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) {
 	msg, err = MetricFamilyToProtobuf(histogramMetricFamily)
 	require.NoError(t, err)
 
+	now = time.Now()
+	total, added, seriesAdded, err = sl.append(app, msg, "application/vnd.google.protobuf", now)
+	require.NoError(t, err)
+	require.Equal(t, 3, total)
+	require.Equal(t, 3, added)
+	require.Equal(t, 3, seriesAdded)
+
+	err = sl.metrics.targetScrapeNativeHistogramBucketLimit.Write(&metric)
+	require.NoError(t, err)
+	metricValue = metric.GetCounter().GetValue()
+	require.Equal(t, beforeMetricValue, metricValue)
+	beforeMetricValue = metricValue
+
+	nativeHistogram.WithLabelValues("L").Observe(100000.0) // in different bucket since > 10*1.1
+
+	gathered, err = registry.Gather()
+	require.NoError(t, err)
+	require.NotEmpty(t, gathered)
+
+	histogramMetricFamily = gathered[0]
+	msg, err = MetricFamilyToProtobuf(histogramMetricFamily)
+	require.NoError(t, err)
+
 	now = time.Now()
 	total, added, seriesAdded, err = sl.append(app, msg, "application/vnd.google.protobuf", now)
 	if !errors.Is(err, errBucketLimit) {

scrape/target.go

@@ -366,13 +366,19 @@ type bucketLimitAppender struct {
 
 func (app *bucketLimitAppender) AppendHistogram(ref storage.SeriesRef, lset labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) {
 	if h != nil {
-		if len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit {
-			return 0, errBucketLimit
+		for len(h.PositiveBuckets)+len(h.NegativeBuckets) > app.limit {
+			if h.Schema == -4 {
+				return 0, errBucketLimit
+			}
+			h = h.ReduceResolution(h.Schema - 1)
 		}
 	}
 	if fh != nil {
-		if len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit {
-			return 0, errBucketLimit
+		for len(fh.PositiveBuckets)+len(fh.NegativeBuckets) > app.limit {
+			if fh.Schema == -4 {
+				return 0, errBucketLimit
+			}
+			fh = fh.ReduceResolution(fh.Schema - 1)
 		}
 	}
 	ref, err := app.Appender.AppendHistogram(ref, lset, t, h, fh)

scrape/target_test.go

@@ -507,10 +507,25 @@ func TestBucketLimitAppender(t *testing.T) {
 		NegativeBuckets: []int64{3, 0, 0},
 	}
 
+	bigGap := histogram.Histogram{
+		Schema:        0,
+		Count:         21,
+		Sum:           33,
+		ZeroThreshold: 0.001,
+		ZeroCount:     3,
+		PositiveSpans: []histogram.Span{
+			{Offset: 1, Length: 1}, // in (1, 2]
+			{Offset: 2, Length: 1}, // in (8, 16]
+		},
+		PositiveBuckets: []int64{1, 0}, // 1, 1
+	}
+
 	cases := []struct {
-		h           histogram.Histogram
-		limit       int
-		expectError bool
+		h                 histogram.Histogram
+		limit             int
+		expectError       bool
+		expectBucketCount int
+		expectSchema      int32
 	}{
 		{
 			h:           example,
@@ -518,9 +533,25 @@ func TestBucketLimitAppender(t *testing.T) {
 			expectError: true,
 		},
 		{
-			h:           example,
-			limit:       10,
-			expectError: false,
+			h:                 example,
+			limit:             4,
+			expectError:       false,
+			expectBucketCount: 4,
+			expectSchema:      -1,
+		},
+		{
+			h:                 example,
+			limit:             10,
+			expectError:       false,
+			expectBucketCount: 6,
+			expectSchema:      0,
+		},
+		{
+			h:                 bigGap,
+			limit:             1,
+			expectError:       false,
+			expectBucketCount: 1,
+			expectSchema:      -2,
 		},
 	}
 
@@ -531,18 +562,28 @@ func TestBucketLimitAppender(t *testing.T) {
 			t.Run(fmt.Sprintf("floatHistogram=%t", floatHisto), func(t *testing.T) {
 				app := &bucketLimitAppender{Appender: resApp, limit: c.limit}
 				ts := int64(10 * time.Minute / time.Millisecond)
-				h := c.h
 				lbls := labels.FromStrings("__name__", "sparse_histogram_series")
 				var err error
 				if floatHisto {
-					_, err = app.AppendHistogram(0, lbls, ts, nil, h.Copy().ToFloat())
-				} else {
-					_, err = app.AppendHistogram(0, lbls, ts, h.Copy(), nil)
-				}
-				if c.expectError {
-					require.Error(t, err)
+					fh := c.h.Copy().ToFloat()
+					_, err = app.AppendHistogram(0, lbls, ts, nil, fh)
+					if c.expectError {
+						require.Error(t, err)
+					} else {
+						require.Equal(t, c.expectSchema, fh.Schema)
+						require.Equal(t, c.expectBucketCount, len(fh.NegativeBuckets)+len(fh.PositiveBuckets))
+						require.NoError(t, err)
+					}
 				} else {
-					require.NoError(t, err)
+					h := c.h.Copy()
+					_, err = app.AppendHistogram(0, lbls, ts, h, nil)
+					if c.expectError {
+						require.Error(t, err)
+					} else {
+						require.Equal(t, c.expectSchema, h.Schema)
+						require.Equal(t, c.expectBucketCount, len(h.NegativeBuckets)+len(h.PositiveBuckets))
+						require.NoError(t, err)
+					}
 				}
 				require.NoError(t, app.Commit())
 			})