apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

test(rw2): add nhcb testcases to remote write 2.0

Browse Source 
Ref: https://github.com/prometheus/prometheus/issues/15021

Also modified spansToSpansProto to not allocate empty bucket spans array
when converting internal model to remote write model.
Otherwise the test TestDecodeWriteV2Request fails since empty array
is marshaled/unmarshaled as nil so we don't get back the exact same
thing.

Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>
pull/16409/head
György Krajcsovits 3 months ago
parent 10b4e1b231
commit 1cdc956d27
3 changed files with 79 additions and 18 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      prompb/io/prometheus/write/v2/codec.go
  2. 62
      storage/remote/codec_test.go
  3. 16
      storage/remote/write_handler_test.go

3
prompb/io/prometheus/write/v2/codec.go
Unescape View File

@ -196,6 +196,9 @@ func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram
}
func spansToSpansProto(s []histogram.Span) []BucketSpan {
if len(s) == 0 {
return nil
}
spans := make([]BucketSpan, len(s))
for i := 0; i < len(s); i++ {
spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length}

62
storage/remote/codec_test.go
Unescape View File

@ -43,12 +43,12 @@ var (
Schema: 2,
ZeroThreshold: 1e-128,
ZeroCount: 0,
Count: 0,
Count: 3,
Sum: 20,
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
PositiveBuckets: []int64{1},
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
NegativeBuckets: []int64{-1},
NegativeBuckets: []int64{2},
}
writeRequestFixture = &prompb.WriteRequest{
@ -90,6 +90,15 @@ var (
Help: "Test counter for test purposes",
}
testHistogramCustomBuckets = histogram.Histogram{
Schema: histogram.CustomBucketsSchema,
Count: 16,
Sum: 20,
PositiveSpans: []histogram.Span{{Offset: 1, Length: 2}},
PositiveBuckets: []int64{10, -4}, // Means 10 observations for upper bound 1.0 and 6 for upper bound +Inf.
CustomValues: []float64{0.1, 1.0}, // +Inf is implied.
}
// writeV2RequestFixture represents the same request as writeRequestFixture,
// but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata.
// NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed.
@ -106,7 +115,12 @@ var (
},
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))},
Histograms: []writev2.Histogram{
writev2.FromIntHistogram(10, &testHistogram),
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
},
CreatedTimestamp: 1, // CT needs to be lower than the sample's timestamp.
},
{
@ -119,12 +133,38 @@ var (
},
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))},
Histograms: []writev2.Histogram{
writev2.FromIntHistogram(50, &testHistogram),
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
},
},
},
}
)
func TestHistogramFixtureValid(t *testing.T) {
for _, ts := range writeRequestFixture.Timeseries {
for _, h := range ts.Histograms {
if h.IsFloatHistogram() {
require.NoError(t, h.ToFloatHistogram().Validate())
} else {
require.NoError(t, h.ToIntHistogram().Validate())
}
}
}
for _, ts := range writeV2RequestFixture.Timeseries {
for _, h := range ts.Histograms {
if h.IsFloatHistogram() {
require.NoError(t, h.ToFloatHistogram().Validate())
} else {
require.NoError(t, h.ToIntHistogram().Validate())
}
}
}
}
func TestWriteV2RequestFixture(t *testing.T) {
// Generate dynamically writeV2RequestFixture, reusing v1 fixture elements.
st := writev2.NewSymbolTable()
@ -143,7 +183,12 @@ func TestWriteV2RequestFixture(t *testing.T) {
},
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))},
Histograms: []writev2.Histogram{
writev2.FromIntHistogram(10, &testHistogram),
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
},
CreatedTimestamp: 1,
},
{
@ -155,7 +200,12 @@ func TestWriteV2RequestFixture(t *testing.T) {
},
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}},
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))},
Histograms: []writev2.Histogram{
writev2.FromIntHistogram(50, &testHistogram),
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
},
},
},
Symbols: st.Symbols(),

16
storage/remote/write_handler_test.go
Unescape View File

@ -391,7 +391,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
desc: "Partial write; first series with one dup histogram sample",
input: func() []writev2.TimeSeries {
f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[1])
f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[len(f.Timeseries[0].Histograms)-1])
return f.Timeseries
}(),
expectedCode: http.StatusBadRequest,
@ -483,7 +483,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
// Double check mandatory 2.0 stats.
// writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each.
expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader))
expectHeaderValue(t, 4, resp.Header.Get(rw20WrittenHistogramsHeader))
expectHeaderValue(t, 8, resp.Header.Get(rw20WrittenHistogramsHeader))
if tc.appendExemplarErr != nil {
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader))
} else {
@ -496,6 +496,8 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
i, j, k, m int
)
for _, ts := range writeV2RequestFixture.Timeseries {
zeroHistogramIngested := false
zeroFloatHistogramIngested := false
ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols)
for _, s := range ts.Samples {
@ -509,21 +511,27 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
for _, hp := range ts.Histograms {
if hp.IsFloatHistogram() {
fh := hp.ToFloatHistogram()
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
if !zeroFloatHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k])
k++
zeroFloatHistogramIngested = true
}
requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
} else {
h := hp.ToIntHistogram()
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
if !zeroHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k])
k++
zeroHistogramIngested = true
}
requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
}
k++
}
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
require.True(t, zeroHistogramIngested)
require.True(t, zeroFloatHistogramIngested)
}
if tc.appendExemplarErr == nil {
for _, e := range ts.Exemplars {
exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels

Write Preview
Loading…
Cancel
Save