Like Prometheus, but for logs.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
loki/pkg/util/evicting_queue_test.go

131 lines
2.3 KiB

Ruler: Recording Rules (#3766) * WIP: hack to get recording rules working and pushing to Cortex/Prometheus Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Refactoring Adding remote_write config for ruler Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Minor refactorings Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Moving manager subpackage into ruler package to avoid dependency cycles This also mirrors Cortex's package structure Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Minor refactorings Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Skipping commit if remote-write client is not defined Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Updating use of cortex client Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Memoizing appenders, using queue for samples & labels Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding buffer size configurability Refactoring Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding metric to show current buffer size Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Refactoring for better responsibility separation & testability Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding per-tenant overrides of remote-write queue capacity Renaming "buffer size" to "queue capacity" to be more accurate Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding tests for evicting queue Minor refactoring Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding more tests and refactoring Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding queue benchmark Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Reducing redundancy in metric names Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Testing that only metric queries can be run Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Minor fixes pre-review Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Appeasing the linter Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Guarding against unprotected nil pointer dereference in Prometheus remote.Client Adding remote-write client validation Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Appeasing the linter Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Setting tenant ID header on remote-write client Adding User-Agent string Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Updating benchmark to use complex struct rather than int to be more reflective of usage Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Registering flags Removing extraneous checks Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Adding metric to track remote-write commit errors Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Refactoring based on review Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Performance improvements based on review Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Return error on invalid queue capacity Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Removing global queue capacity config - using limits Minor refactoring Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Reusing memory in request preparation Refactoring for testability Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Moving remote-write metrics into struct Refactoring Unexporting and refactoring memstore metrics to match Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Applying review suggestions Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Allowing for runtime changing of per-tenant remote-write queue capacity Signed-off-by: Danny Kopping <danny.kopping@grafana.com> * Appeasing the linter Signed-off-by: Danny Kopping <danny.kopping@grafana.com>
4 years ago
package util
import (
"math"
"math/rand"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
func noopOnEvict() {}
func TestQueueAppend(t *testing.T) {
q, err := NewEvictingQueue(10, noopOnEvict)
require.Nil(t, err)
q.Append(1)
q.Append(2)
q.Append(3)
q.Append(4)
q.Append(5)
require.Equal(t, 5, q.Length())
}
func TestQueueCapacity(t *testing.T) {
q, err := NewEvictingQueue(9, noopOnEvict)
require.Nil(t, err)
require.Equal(t, 9, q.Capacity())
q.capacity = 11
require.Equal(t, 11, q.Capacity())
}
func TestZeroCapacityQueue(t *testing.T) {
q, err := NewEvictingQueue(0, noopOnEvict)
require.Error(t, err)
require.Nil(t, q)
}
func TestNegativeCapacityQueue(t *testing.T) {
q, err := NewEvictingQueue(-1, noopOnEvict)
require.Error(t, err)
require.Nil(t, q)
}
func TestQueueEvict(t *testing.T) {
q, err := NewEvictingQueue(3, noopOnEvict)
require.Nil(t, err)
// appending 5 entries will cause the first (oldest) 2 entries to be evicted
entries := []interface{}{1, 2, 3, 4, 5}
for _, entry := range entries {
q.Append(entry)
}
require.Equal(t, 3, q.Length())
require.Equal(t, entries[2:], q.Entries())
}
func TestQueueClear(t *testing.T) {
q, err := NewEvictingQueue(3, noopOnEvict)
require.Nil(t, err)
q.Append(1)
q.Clear()
require.Equal(t, 0, q.Length())
}
func TestQueueEvictionCallback(t *testing.T) {
var evictionCallbackCalls int
q, err := NewEvictingQueue(3, func() {
evictionCallbackCalls++
})
require.Nil(t, err)
for i := 0; i < 5; i++ {
q.Append(i)
}
require.Equal(t, 2, evictionCallbackCalls)
}
func TestSafeConcurrentAccess(t *testing.T) {
q, err := NewEvictingQueue(3, noopOnEvict)
require.Nil(t, err)
var wg sync.WaitGroup
for w := 0; w < 30; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 500; i++ {
q.Append(rand.Int())
}
}()
}
wg.Wait()
require.Equal(t, 3, q.Length())
}
type queueEntry struct {
key string
value interface{}
}
func BenchmarkAppendAndEvict(b *testing.B) {
capacity := 5000
q, err := NewEvictingQueue(capacity, noopOnEvict)
require.Nil(b, err)
b.ResetTimer()
b.ReportAllocs()
for n := 0; n < b.N; n++ {
q.Append(&queueEntry{
key: "hello",
value: "world",
})
}
require.EqualValues(b, math.Min(float64(b.N), float64(capacity)), q.Length())
}