Add circular in-memory exemplars storage (#6635)
* Add circular in-memory exemplars storage Signed-off-by: Callum Styan <callumstyan@gmail.com> Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com> Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> Signed-off-by: Martin Disibio <mdisibio@gmail.com> Co-authored-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> Co-authored-by: Tom Wilkie <tom.wilkie@gmail.com> Co-authored-by: Martin Disibio <mdisibio@gmail.com> * Fix some comments, clean up exemplar metrics struct and exemplar tests. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix exemplar query api null vs empty array issue. Signed-off-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> Co-authored-by: Tom Wilkie <tom.wilkie@gmail.com> Co-authored-by: Martin Disibio <mdisibio@gmail.com>pull/8608/head
Callum Styan
5 years ago
committed by
GitHub
parent
789d49668e
commit
289ba11b79
@ -0,0 +1,209 @@ |
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb |
||||
|
||||
import ( |
||||
"context" |
||||
"sort" |
||||
"sync" |
||||
|
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/prometheus/pkg/exemplar" |
||||
"github.com/prometheus/prometheus/pkg/labels" |
||||
"github.com/prometheus/prometheus/storage" |
||||
) |
||||
|
||||
type CircularExemplarStorage struct { |
||||
outOfOrderExemplars prometheus.Counter |
||||
|
||||
lock sync.RWMutex |
||||
exemplars []*circularBufferEntry |
||||
nextIndex int |
||||
|
||||
// Map of series labels as a string to index entry, which points to the first
|
||||
// and last exemplar for the series in the exemplars circular buffer.
|
||||
index map[string]*indexEntry |
||||
} |
||||
|
||||
type indexEntry struct { |
||||
first int |
||||
last int |
||||
} |
||||
|
||||
type circularBufferEntry struct { |
||||
exemplar exemplar.Exemplar |
||||
seriesLabels labels.Labels |
||||
next int |
||||
} |
||||
|
||||
// If we assume the average case 95 bytes per exemplar we can fit 5651272 exemplars in
|
||||
// 1GB of extra memory, accounting for the fact that this is heap allocated space.
|
||||
// If len < 1, then the exemplar storage is disabled.
|
||||
func NewCircularExemplarStorage(len int, reg prometheus.Registerer) (ExemplarStorage, error) { |
||||
if len < 1 { |
||||
return &noopExemplarStorage{}, nil |
||||
} |
||||
c := &CircularExemplarStorage{ |
||||
exemplars: make([]*circularBufferEntry, len), |
||||
index: make(map[string]*indexEntry), |
||||
outOfOrderExemplars: prometheus.NewCounter(prometheus.CounterOpts{ |
||||
Name: "prometheus_tsdb_exemplar_out_of_order_exemplars_total", |
||||
Help: "Total number of out of order exemplar ingestion failed attempts", |
||||
}), |
||||
} |
||||
|
||||
if reg != nil { |
||||
reg.MustRegister(c.outOfOrderExemplars) |
||||
} |
||||
|
||||
return c, nil |
||||
} |
||||
|
||||
func (ce *CircularExemplarStorage) Appender() *CircularExemplarStorage { |
||||
return ce |
||||
} |
||||
|
||||
func (ce *CircularExemplarStorage) ExemplarQuerier(_ context.Context) (storage.ExemplarQuerier, error) { |
||||
return ce, nil |
||||
} |
||||
|
||||
func (ce *CircularExemplarStorage) Querier(ctx context.Context) (storage.ExemplarQuerier, error) { |
||||
return ce, nil |
||||
} |
||||
|
||||
// Select returns exemplars for a given set of label matchers.
|
||||
func (ce *CircularExemplarStorage) Select(start, end int64, matchers ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { |
||||
ret := make([]exemplar.QueryResult, 0) |
||||
|
||||
ce.lock.RLock() |
||||
defer ce.lock.RUnlock() |
||||
|
||||
// Loop through each index entry, which will point us to first/last exemplar for each series.
|
||||
for _, idx := range ce.index { |
||||
var se exemplar.QueryResult |
||||
e := ce.exemplars[idx.first] |
||||
if !matchesSomeMatcherSet(e.seriesLabels, matchers) { |
||||
continue |
||||
} |
||||
se.SeriesLabels = e.seriesLabels |
||||
|
||||
// Loop through all exemplars in the circular buffer for the current series.
|
||||
for e.exemplar.Ts <= end { |
||||
if e.exemplar.Ts >= start { |
||||
se.Exemplars = append(se.Exemplars, e.exemplar) |
||||
} |
||||
if e.next == -1 { |
||||
break |
||||
} |
||||
e = ce.exemplars[e.next] |
||||
} |
||||
if len(se.Exemplars) > 0 { |
||||
ret = append(ret, se) |
||||
} |
||||
} |
||||
|
||||
sort.Slice(ret, func(i, j int) bool { |
||||
return labels.Compare(ret[i].SeriesLabels, ret[j].SeriesLabels) < 0 |
||||
}) |
||||
|
||||
return ret, nil |
||||
} |
||||
|
||||
func matchesSomeMatcherSet(lbls labels.Labels, matchers [][]*labels.Matcher) bool { |
||||
Outer: |
||||
for _, ms := range matchers { |
||||
for _, m := range ms { |
||||
if !m.Matches(lbls.Get(m.Name)) { |
||||
continue Outer |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// indexGc takes the circularBufferEntry that will be overwritten and updates the
|
||||
// storages index for that entries labelset if necessary.
|
||||
func (ce *CircularExemplarStorage) indexGc(cbe *circularBufferEntry) { |
||||
if cbe == nil { |
||||
return |
||||
} |
||||
|
||||
l := cbe.seriesLabels.String() |
||||
i := cbe.next |
||||
if i == -1 { |
||||
delete(ce.index, l) |
||||
return |
||||
} |
||||
|
||||
ce.index[l] = &indexEntry{i, ce.index[l].last} |
||||
} |
||||
|
||||
func (ce *CircularExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error { |
||||
seriesLabels := l.String() |
||||
ce.lock.Lock() |
||||
defer ce.lock.Unlock() |
||||
|
||||
idx, ok := ce.index[seriesLabels] |
||||
if !ok { |
||||
ce.indexGc(ce.exemplars[ce.nextIndex]) |
||||
// Default the next value to -1 (which we use to detect that we've iterated through all exemplars for a series in Select)
|
||||
// since this is the first exemplar stored for this series.
|
||||
ce.exemplars[ce.nextIndex] = &circularBufferEntry{ |
||||
exemplar: e, |
||||
seriesLabels: l, |
||||
next: -1} |
||||
ce.index[seriesLabels] = &indexEntry{ce.nextIndex, ce.nextIndex} |
||||
ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars) |
||||
return nil |
||||
} |
||||
|
||||
// Check for duplicate vs last stored exemplar for this series.
|
||||
// NB these are expected, add appending them is a no-op.
|
||||
if ce.exemplars[idx.last].exemplar.Equals(e) { |
||||
return nil |
||||
} |
||||
|
||||
if e.Ts <= ce.exemplars[idx.last].exemplar.Ts { |
||||
ce.outOfOrderExemplars.Inc() |
||||
return storage.ErrOutOfOrderExemplar |
||||
} |
||||
ce.indexGc(ce.exemplars[ce.nextIndex]) |
||||
ce.exemplars[ce.nextIndex] = &circularBufferEntry{ |
||||
exemplar: e, |
||||
seriesLabels: l, |
||||
next: -1, |
||||
} |
||||
|
||||
ce.exemplars[ce.index[seriesLabels].last].next = ce.nextIndex |
||||
ce.index[seriesLabels].last = ce.nextIndex |
||||
ce.nextIndex = (ce.nextIndex + 1) % len(ce.exemplars) |
||||
return nil |
||||
} |
||||
|
||||
type noopExemplarStorage struct{} |
||||
|
||||
func (noopExemplarStorage) AddExemplar(l labels.Labels, e exemplar.Exemplar) error { |
||||
return nil |
||||
} |
||||
|
||||
func (noopExemplarStorage) ExemplarQuerier(context.Context) (storage.ExemplarQuerier, error) { |
||||
return &noopExemplarQuerier{}, nil |
||||
} |
||||
|
||||
type noopExemplarQuerier struct{} |
||||
|
||||
func (noopExemplarQuerier) Select(_, _ int64, _ ...[]*labels.Matcher) ([]exemplar.QueryResult, error) { |
||||
return nil, nil |
||||
} |
||||
@ -0,0 +1,316 @@ |
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tsdb |
||||
|
||||
import ( |
||||
"reflect" |
||||
"strconv" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/prometheus/prometheus/pkg/exemplar" |
||||
"github.com/prometheus/prometheus/pkg/labels" |
||||
"github.com/prometheus/prometheus/storage" |
||||
) |
||||
|
||||
func TestAddExemplar(t *testing.T) { |
||||
exs, err := NewCircularExemplarStorage(2, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
e := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "qwerty", |
||||
}, |
||||
}, |
||||
Value: 0.1, |
||||
Ts: 1, |
||||
} |
||||
|
||||
err = es.AddExemplar(l, e) |
||||
require.NoError(t, err) |
||||
require.Equal(t, es.index[l.String()].last, 0, "exemplar was not stored correctly") |
||||
|
||||
e2 := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "zxcvb", |
||||
}, |
||||
}, |
||||
Value: 0.1, |
||||
Ts: 2, |
||||
} |
||||
|
||||
err = es.AddExemplar(l, e2) |
||||
require.NoError(t, err) |
||||
require.Equal(t, es.index[l.String()].last, 1, "exemplar was not stored correctly, location of newest exemplar for series in index did not update") |
||||
require.True(t, es.exemplars[es.index[l.String()].last].exemplar.Equals(e2), "exemplar was not stored correctly, expected %+v got: %+v", e2, es.exemplars[es.index[l.String()].last].exemplar) |
||||
|
||||
err = es.AddExemplar(l, e2) |
||||
require.NoError(t, err, "no error is expected attempting to add duplicate exemplar") |
||||
|
||||
e3 := e2 |
||||
e3.Ts = 3 |
||||
err = es.AddExemplar(l, e3) |
||||
require.NoError(t, err, "no error is expected when attempting to add duplicate exemplar, even with different timestamp") |
||||
|
||||
e3.Ts = 1 |
||||
e3.Value = 0.3 |
||||
err = es.AddExemplar(l, e3) |
||||
require.Equal(t, err, storage.ErrOutOfOrderExemplar) |
||||
} |
||||
|
||||
func TestStorageOverflow(t *testing.T) { |
||||
// Test that circular buffer index and assignment
|
||||
// works properly, adding more exemplars than can
|
||||
// be stored and then querying for them.
|
||||
exs, err := NewCircularExemplarStorage(5, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
|
||||
var eList []exemplar.Exemplar |
||||
for i := 0; i < len(es.exemplars)+1; i++ { |
||||
e := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "a", |
||||
}, |
||||
}, |
||||
Value: float64(i+1) / 10, |
||||
Ts: int64(101 + i), |
||||
} |
||||
es.AddExemplar(l, e) |
||||
eList = append(eList, e) |
||||
} |
||||
require.True(t, (es.exemplars[0].exemplar.Ts == 106), "exemplar was not stored correctly") |
||||
|
||||
m, err := labels.NewMatcher(labels.MatchEqual, l[0].Name, l[0].Value) |
||||
require.NoError(t, err, "error creating label matcher for exemplar query") |
||||
ret, err := es.Select(100, 110, []*labels.Matcher{m}) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
|
||||
require.True(t, reflect.DeepEqual(eList[1:], ret[0].Exemplars), "select did not return expected exemplars\n\texpected: %+v\n\tactual: %+v\n", eList[1:], ret[0].Exemplars) |
||||
} |
||||
|
||||
func TestSelectExemplar(t *testing.T) { |
||||
exs, err := NewCircularExemplarStorage(5, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
e := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "qwerty", |
||||
}, |
||||
}, |
||||
Value: 0.1, |
||||
Ts: 12, |
||||
} |
||||
|
||||
err = es.AddExemplar(l, e) |
||||
require.NoError(t, err, "adding exemplar failed") |
||||
require.True(t, reflect.DeepEqual(es.exemplars[0].exemplar, e), "exemplar was not stored correctly") |
||||
|
||||
m, err := labels.NewMatcher(labels.MatchEqual, l[0].Name, l[0].Value) |
||||
require.NoError(t, err, "error creating label matcher for exemplar query") |
||||
ret, err := es.Select(0, 100, []*labels.Matcher{m}) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
|
||||
expectedResult := []exemplar.Exemplar{e} |
||||
require.True(t, reflect.DeepEqual(expectedResult, ret[0].Exemplars), "select did not return expected exemplars\n\texpected: %+v\n\tactual: %+v\n", expectedResult, ret[0].Exemplars) |
||||
} |
||||
|
||||
func TestSelectExemplar_MultiSeries(t *testing.T) { |
||||
exs, err := NewCircularExemplarStorage(5, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l1 := labels.Labels{ |
||||
{Name: "__name__", Value: "test_metric"}, |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
l2 := labels.Labels{ |
||||
{Name: "__name__", Value: "test_metric2"}, |
||||
{Name: "service", Value: "qwer"}, |
||||
} |
||||
|
||||
for i := 0; i < len(es.exemplars); i++ { |
||||
e1 := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "a", |
||||
}, |
||||
}, |
||||
Value: float64(i+1) / 10, |
||||
Ts: int64(101 + i), |
||||
} |
||||
err = es.AddExemplar(l1, e1) |
||||
require.NoError(t, err) |
||||
|
||||
e2 := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "b", |
||||
}, |
||||
}, |
||||
Value: float64(i+1) / 10, |
||||
Ts: int64(101 + i), |
||||
} |
||||
err = es.AddExemplar(l2, e2) |
||||
require.NoError(t, err) |
||||
} |
||||
|
||||
m, err := labels.NewMatcher(labels.MatchEqual, l2[0].Name, l2[0].Value) |
||||
require.NoError(t, err, "error creating label matcher for exemplar query") |
||||
ret, err := es.Select(100, 200, []*labels.Matcher{m}) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
require.True(t, len(ret[0].Exemplars) == 3, "didn't get expected 8 exemplars, got %d", len(ret[0].Exemplars)) |
||||
|
||||
m, err = labels.NewMatcher(labels.MatchEqual, l1[0].Name, l1[0].Value) |
||||
require.NoError(t, err, "error creating label matcher for exemplar query") |
||||
ret, err = es.Select(100, 200, []*labels.Matcher{m}) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
require.True(t, len(ret[0].Exemplars) == 2, "didn't get expected 8 exemplars, got %d", len(ret[0].Exemplars)) |
||||
} |
||||
|
||||
func TestSelectExemplar_TimeRange(t *testing.T) { |
||||
lenEs := 5 |
||||
exs, err := NewCircularExemplarStorage(lenEs, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
|
||||
for i := 0; i < lenEs; i++ { |
||||
err := es.AddExemplar(l, exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: strconv.Itoa(i), |
||||
}, |
||||
}, |
||||
Value: 0.1, |
||||
Ts: int64(101 + i), |
||||
}) |
||||
require.NoError(t, err) |
||||
require.Equal(t, es.index[l.String()].last, i, "exemplar was not stored correctly") |
||||
} |
||||
|
||||
m, err := labels.NewMatcher(labels.MatchEqual, l[0].Name, l[0].Value) |
||||
require.NoError(t, err, "error creating label matcher for exemplar query") |
||||
ret, err := es.Select(102, 104, []*labels.Matcher{m}) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
require.True(t, len(ret[0].Exemplars) == 3, "didn't get expected two exemplars %d, %+v", len(ret[0].Exemplars), ret) |
||||
} |
||||
|
||||
// Test to ensure that even though a series matches more than one matcher from the
|
||||
// query that it's exemplars are only included in the result a single time.
|
||||
func TestSelectExemplar_DuplicateSeries(t *testing.T) { |
||||
exs, err := NewCircularExemplarStorage(4, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
e := exemplar.Exemplar{ |
||||
Labels: labels.Labels{ |
||||
labels.Label{ |
||||
Name: "traceID", |
||||
Value: "qwerty", |
||||
}, |
||||
}, |
||||
Value: 0.1, |
||||
Ts: 12, |
||||
} |
||||
|
||||
l := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
{Name: "cluster", Value: "us-central1"}, |
||||
} |
||||
|
||||
// Lets just assume somehow the PromQL expression generated two separate lists of matchers,
|
||||
// both of which can select this particular series.
|
||||
m := [][]*labels.Matcher{ |
||||
{ |
||||
labels.MustNewMatcher(labels.MatchEqual, l[0].Name, l[0].Value), |
||||
}, |
||||
{ |
||||
labels.MustNewMatcher(labels.MatchEqual, l[1].Name, l[1].Value), |
||||
}, |
||||
} |
||||
|
||||
err = es.AddExemplar(l, e) |
||||
require.NoError(t, err, "adding exemplar failed") |
||||
require.True(t, reflect.DeepEqual(es.exemplars[0].exemplar, e), "exemplar was not stored correctly") |
||||
|
||||
ret, err := es.Select(0, 100, m...) |
||||
require.NoError(t, err) |
||||
require.True(t, len(ret) == 1, "select should have returned samples for a single series only") |
||||
} |
||||
|
||||
func TestIndexOverwrite(t *testing.T) { |
||||
exs, err := NewCircularExemplarStorage(2, nil) |
||||
require.NoError(t, err) |
||||
es := exs.(*CircularExemplarStorage) |
||||
|
||||
l1 := labels.Labels{ |
||||
{Name: "service", Value: "asdf"}, |
||||
} |
||||
|
||||
l2 := labels.Labels{ |
||||
{Name: "service", Value: "qwer"}, |
||||
} |
||||
|
||||
err = es.AddExemplar(l1, exemplar.Exemplar{Value: 1, Ts: 1}) |
||||
require.NoError(t, err) |
||||
err = es.AddExemplar(l2, exemplar.Exemplar{Value: 2, Ts: 2}) |
||||
require.NoError(t, err) |
||||
err = es.AddExemplar(l2, exemplar.Exemplar{Value: 3, Ts: 3}) |
||||
require.NoError(t, err) |
||||
|
||||
// Ensure index GC'ing is taking place, there should no longer be any
|
||||
// index entry for series l1 since we just wrote two exemplars for series l2.
|
||||
_, ok := es.index[l1.String()] |
||||
require.False(t, ok) |
||||
require.Equal(t, &indexEntry{1, 0}, es.index[l2.String()]) |
||||
|
||||
err = es.AddExemplar(l1, exemplar.Exemplar{Value: 4, Ts: 4}) |
||||
require.NoError(t, err) |
||||
|
||||
i := es.index[l2.String()] |
||||
require.Equal(t, &indexEntry{0, 0}, i) |
||||
} |
||||
Loading…
Reference in new issue