apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

textparse/scrape: Add option to scrape both classic and native histograms

Browse Source 
So far, if a target exposes a histogram with both classic and native
buckets, a native-histogram enabled Prometheus would ignore the
classic buckets. With the new scrape config option
`scrape_classic_histograms` set, both buckets will be ingested,
creating all the series of a classic histogram in parallel to the
native histogram series. For example, a histogram `foo` would create a
native histogram series `foo` and classic series called `foo_sum`,
`foo_count`, and `foo_bucket`.

This feature can be used in a migration strategy from classic to
native histograms, where it is desired to have a transition period
during which both native and classic histograms are present.

Note that two bugs in classic histogram parsing were found and fixed
as a byproduct of testing the new feature:

1. Series created from classic _gauge_ histograms didn't get the
   _sum/_count/_bucket prefix set.
2. Values of classic _float_ histograms weren't parsed properly.

Signed-off-by: beorn7 <beorn@grafana.com>
pull/12350/head
beorn7 3 years ago
parent bd98fc8c45
commit 9e500345f3
10 changed files with 1152 additions and 490 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 7
      config/config.go
  2. 4
      docs/configuration/configuration.md
  3. 6
      model/textparse/interface.go
  4. 2
      model/textparse/interface_test.go
  5. 31
      model/textparse/protobufparse.go
  6. 617
      model/textparse/protobufparse_test.go
  7. 2
      promql/fuzz.go
  8. 9
      scrape/scrape.go
  9. 36
      scrape/scrape_test.go
  10. 2
      web/federate_test.go

7
config/config.go
Unescape View File

@ -146,8 +146,9 @@ var (
// DefaultScrapeConfig is the default scrape configuration.
DefaultScrapeConfig = ScrapeConfig{
// ScrapeTimeout and ScrapeInterval default to the
// configured globals.
// ScrapeTimeout and ScrapeInterval default to the configured
// globals.
ScrapeClassicHistograms: false,
MetricsPath: "/metrics",
Scheme: "http",
HonorLabels: false,
@ -467,6 +468,8 @@ type ScrapeConfig struct {
ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
// The timeout for scraping targets of this config.
ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
// Whether to scrape a classic histogram that is also exposed as a native histogram.
ScrapeClassicHistograms bool `yaml:"scrape_classic_histograms,omitempty"`
// The HTTP resource path on which to fetch metrics from targets.
MetricsPath string `yaml:"metrics_path,omitempty"`
// The URL scheme with which to fetch metrics from targets.

4
docs/configuration/configuration.md
Unescape View File

@ -134,6 +134,10 @@ job_name: <job_name>
# Per-scrape timeout when scraping this job.
[ scrape_timeout: <duration> | default = <global_config.scrape_timeout> ]
# Whether to scrape a classic histogram that is also exposed as a native
# histogram (has no effect without --enable-feature=native-histograms).
[ scrape_classic_histograms: <boolean> | default = false ]
# The HTTP resource path on which to fetch metrics from targets.
[ metrics_path: <path> | default = /metrics ]

6
model/textparse/interface.go
Unescape View File

@ -71,7 +71,7 @@ type Parser interface {
//
// This function always returns a valid parser, but might additionally
// return an error if the content type cannot be parsed.
func New(b []byte, contentType string) (Parser, error) {
func New(b []byte, contentType string, parseClassicHistograms bool) (Parser, error) {
if contentType == "" {
return NewPromParser(b), nil
}
@ -84,7 +84,7 @@ func New(b []byte, contentType string) (Parser, error) {
case "application/openmetrics-text":
return NewOpenMetricsParser(b), nil
case "application/vnd.google.protobuf":
return NewProtobufParser(b), nil
return NewProtobufParser(b, parseClassicHistograms), nil
default:
return NewPromParser(b), nil
}
@ -100,7 +100,7 @@ const (
EntrySeries Entry = 2 // A series with a simple float64 as value.
EntryComment Entry = 3
EntryUnit Entry = 4
EntryHistogram Entry = 5 // A series with a sparse histogram as a value.
EntryHistogram Entry = 5 // A series with a native histogram as a value.
)
// MetricType represents metric type values.

2
model/textparse/interface_test.go
Unescape View File

@ -91,7 +91,7 @@ func TestNewParser(t *testing.T) {
tt := tt // Copy to local variable before going parallel.
t.Parallel()
p, err := New([]byte{}, tt.contentType)
p, err := New([]byte{}, tt.contentType, false)
tt.validateParser(t, p)
if tt.err == "" {
require.NoError(t, err)

31
model/textparse/protobufparse.go
Unescape View File

@ -54,6 +54,8 @@ type ProtobufParser struct {
// quantiles/buckets.
fieldPos int
fieldsDone bool // true if no more fields of a Summary or (legacy) Histogram to be processed.
redoClassic bool // true after parsing a native histogram if we need to parse it again as a classit histogram.
// state is marked by the entry we are processing. EntryInvalid implies
// that we have to decode the next MetricFamily.
state Entry
@ -62,17 +64,22 @@ type ProtobufParser struct {
mf *dto.MetricFamily
// Wether to also parse a classic histogram that is also present as a
// native histogram.
parseClassicHistograms bool
// The following are just shenanigans to satisfy the Parser interface.
metricBytes *bytes.Buffer // A somewhat fluid representation of the current metric.
}
// NewProtobufParser returns a parser for the payload in the byte slice.
func NewProtobufParser(b []byte) Parser {
func NewProtobufParser(b []byte, parseClassicHistograms bool) Parser {
return &ProtobufParser{
in: b,
state: EntryInvalid,
mf: &dto.MetricFamily{},
metricBytes: &bytes.Buffer{},
parseClassicHistograms: parseClassicHistograms,
}
}
@ -106,21 +113,30 @@ func (p *ProtobufParser) Series() ([]byte, *int64, float64) {
v = s.GetQuantile()[p.fieldPos].GetValue()
}
case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM:
// This should only happen for a legacy histogram.
// This should only happen for a classic histogram.
h := m.GetHistogram()
switch p.fieldPos {
case -2:
v = h.GetSampleCountFloat()
if v == 0 {
v = float64(h.GetSampleCount())
}
case -1:
v = h.GetSampleSum()
default:
bb := h.GetBucket()
if p.fieldPos >= len(bb) {
v = h.GetSampleCountFloat()
if v == 0 {
v = float64(h.GetSampleCount())
}
} else {
v = bb[p.fieldPos].GetCumulativeCountFloat()
if v == 0 {
v = float64(bb[p.fieldPos].GetCumulativeCount())
}
}
}
default:
panic("encountered unexpected metric type, this is a bug")
}
@ -149,6 +165,9 @@ func (p *ProtobufParser) Histogram() ([]byte, *int64, *histogram.Histogram, *his
ts = m.GetTimestampMs()
h = m.GetHistogram()
)
if p.parseClassicHistograms && len(h.GetBucket()) > 0 {
p.redoClassic = true
}
if h.GetSampleCountFloat() > 0 || h.GetZeroCountFloat() > 0 {
// It is a float histogram.
fh := histogram.FloatHistogram{
@ -376,6 +395,12 @@ func (p *ProtobufParser) Next() (Entry, error) {
return EntryInvalid, err
}
case EntryHistogram, EntrySeries:
if p.redoClassic {
p.redoClassic = false
p.state = EntrySeries
p.fieldPos = -3
p.fieldsDone = false
}
t := p.mf.GetType()
if p.state == EntrySeries && !p.fieldsDone &&
(t == dto.MetricType_SUMMARY ||
@ -432,7 +457,7 @@ func (p *ProtobufParser) updateMetricBytes() error {
// state.
func (p *ProtobufParser) getMagicName() string {
t := p.mf.GetType()
if p.state == EntryHistogram || (t != dto.MetricType_HISTOGRAM && t != dto.MetricType_SUMMARY) {
if p.state == EntryHistogram || (t != dto.MetricType_HISTOGRAM && t != dto.MetricType_GAUGE_HISTOGRAM && t != dto.MetricType_SUMMARY) {
return p.mf.GetName()
}
if p.fieldPos == -2 {

617
model/textparse/protobufparse_test.go
Unescape View File

@ -30,8 +30,8 @@ import (
dto "github.com/prometheus/prometheus/prompb/io/prometheus/client"
)
func TestProtobufParse(t *testing.T) {
textMetricFamilies := []string{
func createTestProtoBuf(t *testing.T) *bytes.Buffer {
testMetricFamilies := []string{
`name: "go_build_info"
help: "Build information about the main Go module."
type: GAUGE
@ -231,7 +231,6 @@ help: "Test float histogram with many buckets removed to keep it manageable in s
type: HISTOGRAM
metric: <
histogram: <
sample_count: 175
sample_count_float: 175.0
sample_sum: 0.0008280461746287094
bucket: <
@ -302,7 +301,6 @@ help: "Like test_float_histogram but as gauge histogram."
type: GAUGE_HISTOGRAM
metric: <
histogram: <
sample_count: 175
sample_count_float: 175.0
sample_sum: 0.0008280461746287094
bucket: <
@ -450,9 +448,9 @@ metric: <
}
varintBuf := make([]byte, binary.MaxVarintLen32)
inputBuf := &bytes.Buffer{}
buf := &bytes.Buffer{}
for _, tmf := range textMetricFamilies {
for _, tmf := range testMetricFamilies {
pb := &dto.MetricFamily{}
// From text to proto message.
require.NoError(t, proto.UnmarshalText(tmf, pb))
@ -462,11 +460,15 @@ metric: <
// Write first length, then binary protobuf.
varintLength := binary.PutUvarint(varintBuf, uint64(len(protoBuf)))
inputBuf.Write(varintBuf[:varintLength])
inputBuf.Write(protoBuf)
buf.Write(varintBuf[:varintLength])
buf.Write(protoBuf)
}
exp := []struct {
return buf
}
func TestProtobufParse(t *testing.T) {
type parseResult struct {
lset labels.Labels
m string
t int64
@ -478,7 +480,19 @@ metric: <
shs *histogram.Histogram
fhs *histogram.FloatHistogram
e []exemplar.Exemplar
}
inputBuf := createTestProtoBuf(t)
scenarios := []struct {
name string
parser Parser
expected []parseResult
}{
{
name: "ignore classic buckets of native histograms",
parser: NewProtobufParser(inputBuf.Bytes(), false),
expected: []parseResult{
{
m: "go_build_info",
help: "Build information about the main Go module.",
@ -810,12 +824,587 @@ metric: <
"__name__", "without_quantiles_sum",
),
},
},
},
{
name: "parse classic and native buckets",
parser: NewProtobufParser(inputBuf.Bytes(), true),
expected: []parseResult{
{ // 0
m: "go_build_info",
help: "Build information about the main Go module.",
},
{ // 1
m: "go_build_info",
typ: MetricTypeGauge,
},
{ // 2
m: "go_build_info\xFFchecksum\xFF\xFFpath\xFFgithub.com/prometheus/client_golang\xFFversion\xFF(devel)",
v: 1,
lset: labels.FromStrings(
"__name__", "go_build_info",
"checksum", "",
"path", "github.com/prometheus/client_golang",
"version", "(devel)",
),
},
{ // 3
m: "go_memstats_alloc_bytes_total",
help: "Total number of bytes allocated, even if freed.",
},
{ // 4
m: "go_memstats_alloc_bytes_total",
typ: MetricTypeCounter,
},
{ // 5
m: "go_memstats_alloc_bytes_total",
v: 1.546544e+06,
lset: labels.FromStrings(
"__name__", "go_memstats_alloc_bytes_total",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "42"), Value: 12, HasTs: true, Ts: 1625851151233},
},
},
{ // 6
m: "something_untyped",
help: "Just to test the untyped type.",
},
{ // 7
m: "something_untyped",
typ: MetricTypeUnknown,
},
{ // 8
m: "something_untyped",
t: 1234567,
v: 42,
lset: labels.FromStrings(
"__name__", "something_untyped",
),
},
{ // 9
m: "test_histogram",
help: "Test histogram with many buckets removed to keep it manageable in size.",
},
{ // 10
m: "test_histogram",
typ: MetricTypeHistogram,
},
{ // 11
m: "test_histogram",
t: 1234568,
shs: &histogram.Histogram{
Count: 175,
ZeroCount: 2,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []int64{1, 2, -1, -1},
NegativeBuckets: []int64{1, 3, -2, -1, 1},
},
lset: labels.FromStrings(
"__name__", "test_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 12
m: "test_histogram_count",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram_count",
),
},
{ // 13
m: "test_histogram_sum",
t: 1234568,
v: 0.0008280461746287094,
lset: labels.FromStrings(
"__name__", "test_histogram_sum",
),
},
{ // 14
m: "test_histogram_bucket\xffle\xff-0.0004899999999999998",
t: 1234568,
v: 2,
lset: labels.FromStrings(
"__name__", "test_histogram_bucket",
"le", "-0.0004899999999999998",
),
},
{ // 15
m: "test_histogram_bucket\xffle\xff-0.0003899999999999998",
t: 1234568,
v: 4,
lset: labels.FromStrings(
"__name__", "test_histogram_bucket",
"le", "-0.0003899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
},
},
{ // 16
m: "test_histogram_bucket\xffle\xff-0.0002899999999999998",
t: 1234568,
v: 16,
lset: labels.FromStrings(
"__name__", "test_histogram_bucket",
"le", "-0.0002899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 17
m: "test_histogram_bucket\xffle\xff+Inf",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram_bucket",
"le", "+Inf",
),
},
{ // 18
m: "test_gauge_histogram",
help: "Like test_histogram but as gauge histogram.",
},
{ // 19
m: "test_gauge_histogram",
typ: MetricTypeGaugeHistogram,
},
{ // 20
m: "test_gauge_histogram",
t: 1234568,
shs: &histogram.Histogram{
CounterResetHint: histogram.GaugeType,
Count: 175,
ZeroCount: 2,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []int64{1, 2, -1, -1},
NegativeBuckets: []int64{1, 3, -2, -1, 1},
},
lset: labels.FromStrings(
"__name__", "test_gauge_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 21
m: "test_gauge_histogram_count",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_count",
),
},
{ // 22
m: "test_gauge_histogram_sum",
t: 1234568,
v: 0.0008280461746287094,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_sum",
),
},
{ // 23
m: "test_gauge_histogram_bucket\xffle\xff-0.0004899999999999998",
t: 1234568,
v: 2,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_bucket",
"le", "-0.0004899999999999998",
),
},
{ // 24
m: "test_gauge_histogram_bucket\xffle\xff-0.0003899999999999998",
t: 1234568,
v: 4,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_bucket",
"le", "-0.0003899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
},
},
{ // 25
m: "test_gauge_histogram_bucket\xffle\xff-0.0002899999999999998",
t: 1234568,
v: 16,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_bucket",
"le", "-0.0002899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 26
m: "test_gauge_histogram_bucket\xffle\xff+Inf",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_gauge_histogram_bucket",
"le", "+Inf",
),
},
{ // 27
m: "test_float_histogram",
help: "Test float histogram with many buckets removed to keep it manageable in size.",
},
{ // 28
m: "test_float_histogram",
typ: MetricTypeHistogram,
},
{ // 29
m: "test_float_histogram",
t: 1234568,
fhs: &histogram.FloatHistogram{
Count: 175.0,
ZeroCount: 2.0,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
},
lset: labels.FromStrings(
"__name__", "test_float_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 30
m: "test_float_histogram_count",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_float_histogram_count",
),
},
{ // 31
m: "test_float_histogram_sum",
t: 1234568,
v: 0.0008280461746287094,
lset: labels.FromStrings(
"__name__", "test_float_histogram_sum",
),
},
{ // 32
m: "test_float_histogram_bucket\xffle\xff-0.0004899999999999998",
t: 1234568,
v: 2,
lset: labels.FromStrings(
"__name__", "test_float_histogram_bucket",
"le", "-0.0004899999999999998",
),
},
{ // 33
m: "test_float_histogram_bucket\xffle\xff-0.0003899999999999998",
t: 1234568,
v: 4,
lset: labels.FromStrings(
"__name__", "test_float_histogram_bucket",
"le", "-0.0003899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
},
},
{ // 34
m: "test_float_histogram_bucket\xffle\xff-0.0002899999999999998",
t: 1234568,
v: 16,
lset: labels.FromStrings(
"__name__", "test_float_histogram_bucket",
"le", "-0.0002899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 35
m: "test_float_histogram_bucket\xffle\xff+Inf",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_float_histogram_bucket",
"le", "+Inf",
),
},
{ // 36
m: "test_gauge_float_histogram",
help: "Like test_float_histogram but as gauge histogram.",
},
{ // 37
m: "test_gauge_float_histogram",
typ: MetricTypeGaugeHistogram,
},
{ // 38
m: "test_gauge_float_histogram",
t: 1234568,
fhs: &histogram.FloatHistogram{
CounterResetHint: histogram.GaugeType,
Count: 175.0,
ZeroCount: 2.0,
Sum: 0.0008280461746287094,
ZeroThreshold: 2.938735877055719e-39,
Schema: 3,
PositiveSpans: []histogram.Span{
{Offset: -161, Length: 1},
{Offset: 8, Length: 3},
},
NegativeSpans: []histogram.Span{
{Offset: -162, Length: 1},
{Offset: 23, Length: 4},
},
PositiveBuckets: []float64{1.0, 2.0, -1.0, -1.0},
NegativeBuckets: []float64{1.0, 3.0, -2.0, -1.0, 1.0},
},
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 39
m: "test_gauge_float_histogram_count",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_count",
),
},
{ // 40
m: "test_gauge_float_histogram_sum",
t: 1234568,
v: 0.0008280461746287094,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_sum",
),
},
{ // 41
m: "test_gauge_float_histogram_bucket\xffle\xff-0.0004899999999999998",
t: 1234568,
v: 2,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_bucket",
"le", "-0.0004899999999999998",
),
},
{ // 42
m: "test_gauge_float_histogram_bucket\xffle\xff-0.0003899999999999998",
t: 1234568,
v: 4,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_bucket",
"le", "-0.0003899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00039, HasTs: true, Ts: 1625851155146},
},
},
{ // 43
m: "test_gauge_float_histogram_bucket\xffle\xff-0.0002899999999999998",
t: 1234568,
v: 16,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_bucket",
"le", "-0.0002899999999999998",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.00029, HasTs: false},
},
},
{ // 44
m: "test_gauge_float_histogram_bucket\xffle\xff+Inf",
t: 1234568,
v: 175,
lset: labels.FromStrings(
"__name__", "test_gauge_float_histogram_bucket",
"le", "+Inf",
),
},
{ // 45
m: "test_histogram2",
help: "Similar histogram as before but now without sparse buckets.",
},
{ // 46
m: "test_histogram2",
typ: MetricTypeHistogram,
},
{ // 47
m: "test_histogram2_count",
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram2_count",
),
},
{ // 48
m: "test_histogram2_sum",
v: 0.000828,
lset: labels.FromStrings(
"__name__", "test_histogram2_sum",
),
},
{ // 49
m: "test_histogram2_bucket\xffle\xff-0.00048",
v: 2,
lset: labels.FromStrings(
"__name__", "test_histogram2_bucket",
"le", "-0.00048",
),
},
{ // 50
m: "test_histogram2_bucket\xffle\xff-0.00038",
v: 4,
lset: labels.FromStrings(
"__name__", "test_histogram2_bucket",
"le", "-0.00038",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "59727"), Value: -0.00038, HasTs: true, Ts: 1625851153146},
},
},
{ // 51
m: "test_histogram2_bucket\xffle\xff1.0",
v: 16,
lset: labels.FromStrings(
"__name__", "test_histogram2_bucket",
"le", "1.0",
),
e: []exemplar.Exemplar{
{Labels: labels.FromStrings("dummyID", "5617"), Value: -0.000295, HasTs: false},
},
},
{ // 52
m: "test_histogram2_bucket\xffle\xff+Inf",
v: 175,
lset: labels.FromStrings(
"__name__", "test_histogram2_bucket",
"le", "+Inf",
),
},
{ // 53
m: "rpc_durations_seconds",
help: "RPC latency distributions.",
},
{ // 54
m: "rpc_durations_seconds",
typ: MetricTypeSummary,
},
{ // 55
m: "rpc_durations_seconds_count\xffservice\xffexponential",
v: 262,
lset: labels.FromStrings(
"__name__", "rpc_durations_seconds_count",
"service", "exponential",
),
},
{ // 56
m: "rpc_durations_seconds_sum\xffservice\xffexponential",
v: 0.00025551262820703587,
lset: labels.FromStrings(
"__name__", "rpc_durations_seconds_sum",
"service", "exponential",
),
},
{ // 57
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.5",
v: 6.442786329648548e-07,
lset: labels.FromStrings(
"__name__", "rpc_durations_seconds",
"quantile", "0.5",
"service", "exponential",
),
},
{ // 58
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.9",
v: 1.9435742936658396e-06,
lset: labels.FromStrings(
"__name__", "rpc_durations_seconds",
"quantile", "0.9",
"service", "exponential",
),
},
{ // 59
m: "rpc_durations_seconds\xffservice\xffexponential\xffquantile\xff0.99",
v: 4.0471608667037015e-06,
lset: labels.FromStrings(
"__name__", "rpc_durations_seconds",
"quantile", "0.99",
"service", "exponential",
),
},
{ // 60
m: "without_quantiles",
help: "A summary without quantiles.",
},
{ // 61
m: "without_quantiles",
typ: MetricTypeSummary,
},
{ // 62
m: "without_quantiles_count",
v: 42,
lset: labels.FromStrings(
"__name__", "without_quantiles_count",
),
},
{ // 63
m: "without_quantiles_sum",
v: 1.234,
lset: labels.FromStrings(
"__name__", "without_quantiles_sum",
),
},
},
},
}
p := NewProtobufParser(inputBuf.Bytes())
i := 0
var res labels.Labels
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
var (
i int
res labels.Labels
p = scenario.parser
exp = scenario.expected
)
for {
et, err := p.Next()
@ -891,4 +1480,6 @@ metric: <
i++
}
require.Equal(t, len(exp), i)
})
}
}

2
promql/fuzz.go
Unescape View File

@ -58,7 +58,7 @@ const (
)
func fuzzParseMetricWithContentType(in []byte, contentType string) int {
p, warning := textparse.New(in, contentType)
p, warning := textparse.New(in, contentType, false)
if warning != nil {
// An invalid content type is being passed, which should not happen
// in this context.

9
scrape/scrape.go
Unescape View File

@ -269,6 +269,7 @@ type scrapeLoopOptions struct {
honorTimestamps bool
interval time.Duration
timeout time.Duration
scrapeClassicHistograms bool
mrc []*relabel.Config
cache *scrapeCache
}
@ -331,6 +332,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app storage.Appendable, jitterSeed
opts.labelLimits,
opts.interval,
opts.timeout,
opts.scrapeClassicHistograms,
options.ExtraMetrics,
options.EnableMetadataStorage,
opts.target,
@ -550,6 +552,7 @@ func (sp *scrapePool) sync(targets []*Target) {
honorLabels = sp.config.HonorLabels
honorTimestamps = sp.config.HonorTimestamps
mrc = sp.config.MetricRelabelConfigs
scrapeClassicHistograms = sp.config.ScrapeClassicHistograms
)
sp.targetMtx.Lock()
@ -578,6 +581,7 @@ func (sp *scrapePool) sync(targets []*Target) {
mrc: mrc,
interval: interval,
timeout: timeout,
scrapeClassicHistograms: scrapeClassicHistograms,
})
if err != nil {
l.setForcedError(err)
@ -896,6 +900,7 @@ type scrapeLoop struct {
labelLimits *labelLimits
interval time.Duration
timeout time.Duration
scrapeClassicHistograms bool
appender func(ctx context.Context) storage.Appender
sampleMutator labelsMutator
@ -1177,6 +1182,7 @@ func newScrapeLoop(ctx context.Context,
labelLimits *labelLimits,
interval time.Duration,
timeout time.Duration,
scrapeClassicHistograms bool,
reportExtraMetrics bool,
appendMetadataToWAL bool,
target *Target,
@ -1221,6 +1227,7 @@ func newScrapeLoop(ctx context.Context,
labelLimits: labelLimits,
interval: interval,
timeout: timeout,
scrapeClassicHistograms: scrapeClassicHistograms,
reportExtraMetrics: reportExtraMetrics,
appendMetadataToWAL: appendMetadataToWAL,
}
@ -1492,7 +1499,7 @@ type appendErrors struct {
}
func (sl *scrapeLoop) append(app storage.Appender, b []byte, contentType string, ts time.Time) (total, added, seriesAdded int, err error) {
p, err := textparse.New(b, contentType)
p, err := textparse.New(b, contentType, sl.scrapeClassicHistograms)
if err != nil {
level.Debug(sl.l).Log(
"msg", "Invalid content type on scrape, using prometheus parser as fallback.",

36
scrape/scrape_test.go
Unescape View File

@ -633,6 +633,7 @@ func TestScrapeLoopStopBeforeRun(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -705,6 +706,7 @@ func TestScrapeLoopStop(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -781,6 +783,7 @@ func TestScrapeLoopRun(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -836,6 +839,7 @@ func TestScrapeLoopRun(t *testing.T) {
100*time.Millisecond,
false,
false,
false,
nil,
false,
)
@ -895,6 +899,7 @@ func TestScrapeLoopForcedErr(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -953,6 +958,7 @@ func TestScrapeLoopMetadata(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1010,6 +1016,7 @@ func simpleTestScrapeLoop(t testing.TB) (context.Context, *scrapeLoop) {
0,
false,
false,
false,
nil,
false,
)
@ -1070,6 +1077,7 @@ func TestScrapeLoopFailWithInvalidLabelsAfterRelabel(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1148,6 +1156,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnFailedScrape(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -1211,6 +1220,7 @@ func TestScrapeLoopRunCreatesStaleMarkersOnParseFailure(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -1277,6 +1287,7 @@ func TestScrapeLoopCache(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -1360,6 +1371,7 @@ func TestScrapeLoopCacheMemoryExhaustionProtection(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -1474,6 +1486,7 @@ func TestScrapeLoopAppend(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1563,7 +1576,7 @@ func TestScrapeLoopAppendForConflictingPrefixedLabels(t *testing.T) {
return mutateSampleLabels(l, &Target{labels: labels.FromStrings(tc.targetLabels...)}, false, nil)
},
nil,
func(ctx context.Context) storage.Appender { return app }, nil, 0, true, 0, 0, nil, 0, 0, false, false, nil, false,
func(ctx context.Context) storage.Appender { return app }, nil, 0, true, 0, 0, nil, 0, 0, false, false, false, nil, false,
)
slApp := sl.appender(context.Background())
_, _, _, err := sl.append(slApp, []byte(tc.exposedLabels), "", time.Date(2000, 1, 1, 1, 0, 0, 0, time.UTC))
@ -1600,6 +1613,7 @@ func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1607,7 +1621,7 @@ func TestScrapeLoopAppendCacheEntryButErrNotFound(t *testing.T) {
fakeRef := storage.SeriesRef(1)
expValue := float64(1)
metric := []byte(`metric{n="1"} 1`)
p, warning := textparse.New(metric, "")
p, warning := textparse.New(metric, "", false)
require.NoError(t, warning)
var lset labels.Labels
@ -1658,6 +1672,7 @@ func TestScrapeLoopAppendSampleLimit(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1735,6 +1750,7 @@ func TestScrapeLoop_HistogramBucketLimit(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1833,6 +1849,7 @@ func TestScrapeLoop_ChangingMetricString(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1881,6 +1898,7 @@ func TestScrapeLoopAppendStaleness(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -1932,6 +1950,7 @@ func TestScrapeLoopAppendNoStalenessIfTimestamp(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2043,6 +2062,7 @@ metric_total{n="2"} 2 # {t="2"} 2.0 20000
0,
false,
false,
false,
nil,
false,
)
@ -2108,6 +2128,7 @@ func TestScrapeLoopAppendExemplarSeries(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2160,6 +2181,7 @@ func TestScrapeLoopRunReportsTargetDownOnScrapeError(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -2196,6 +2218,7 @@ func TestScrapeLoopRunReportsTargetDownOnInvalidUTF8(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -2245,6 +2268,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T
0,
false,
false,
false,
nil,
false,
)
@ -2290,6 +2314,7 @@ func TestScrapeLoopOutOfBoundsTimeError(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2562,6 +2587,7 @@ func TestScrapeLoop_RespectTimestamps(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2603,6 +2629,7 @@ func TestScrapeLoop_DiscardTimestamps(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2643,6 +2670,7 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2701,6 +2729,7 @@ func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -2964,6 +2993,7 @@ func TestScrapeAddFast(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)
@ -3050,6 +3080,7 @@ func TestScrapeReportSingleAppender(t *testing.T) {
time.Hour,
false,
false,
false,
nil,
false,
)
@ -3252,6 +3283,7 @@ func TestScrapeLoopLabelLimit(t *testing.T) {
0,
false,
false,
false,
nil,
false,
)

2
web/federate_test.go
Unescape View File

@ -378,7 +378,7 @@ func TestFederationWithNativeHistograms(t *testing.T) {
body, err := io.ReadAll(res.Body)
require.NoError(t, err)
p := textparse.NewProtobufParser(body)
p := textparse.NewProtobufParser(body, false)
var actVec promql.Vector
metricFamilies := 0
l := labels.Labels{}

Write Preview
Loading…
Cancel
Save