Like Prometheus, but for logs.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
loki/pkg/querier/queryrange/ordering.go

98 lines
2.3 KiB

Loki Query Frontend (#1442) * Adds frontend to Loki. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improves tests. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes sneaky bug in entries sorting. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes the split by interval. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Tweak jsonnet deployments and add a way to lint/fmt. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * lint. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * fix timezone issue. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improve tests and rollback change in loghttp package. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes a flaky test that might run one more goroutine. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes windows build. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Improve tracing in the split by interval. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add test stream to proto conversion. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Fixes flappy retry test. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Remove err shadowing in stopQueryFrontend as it was confusing. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Refactor grpc message size in the libsonnet config file. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Don't check auth header for GRPC TransferChunks. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Query frontend (#3) * frontend codec merging optimizations * codec benchmarks * removes unused bounds code in queryrange ordering * [wip] splitby uses channels instead of sub batching intervals * splitBy channel limit test * single allocation for merging entries from a single stream * skip merging loki responses when limit is already hit * removes checks for unlimited queries in queryrange * removes splitByInterval{,.interval} spans * removes interval_batch_size from jsonnet lib * moves benchmark utils to own file * renames markers -> entries * priority queue comments * Removes unused logRequest. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Sets the cache interval to the same split interval. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Missing import libsonnet for the frontend. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Frontend should not be a cluster IP. Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> Co-authored-by: Owen Diehl <ow.diehl@gmail.com>
5 years ago
package queryrange
import (
"sort"
"github.com/grafana/loki/pkg/logproto"
)
/*
Utils for manipulating ordering
*/
type entries []logproto.Entry
func (m entries) start() int64 {
if len(m) == 0 {
return 0
}
return m[0].Timestamp.UnixNano()
}
type byDir struct {
markers []entries
direction logproto.Direction
labels string
}
func (a byDir) Len() int { return len(a.markers) }
func (a byDir) Swap(i, j int) { a.markers[i], a.markers[j] = a.markers[j], a.markers[i] }
func (a byDir) Less(i, j int) bool {
x, y := a.markers[i].start(), a.markers[j].start()
if a.direction == logproto.BACKWARD {
return x > y
}
return y > x
}
func (a byDir) EntriesCount() (n int) {
for _, m := range a.markers {
n += len(m)
}
return n
}
func (a byDir) merge() []logproto.Entry {
result := make([]logproto.Entry, 0, a.EntriesCount())
sort.Sort(a)
for _, m := range a.markers {
result = append(result, m...)
}
return result
}
// priorityqueue is used for extracting a limited # of entries from a set of sorted streams
type priorityqueue struct {
streams []*logproto.Stream
direction logproto.Direction
}
func (pq *priorityqueue) Len() int { return len(pq.streams) }
func (pq *priorityqueue) Less(i, j int) bool {
if pq.direction == logproto.FORWARD {
return pq.streams[i].Entries[0].Timestamp.UnixNano() < pq.streams[j].Entries[0].Timestamp.UnixNano()
}
return pq.streams[i].Entries[0].Timestamp.UnixNano() > pq.streams[j].Entries[0].Timestamp.UnixNano()
}
func (pq *priorityqueue) Swap(i, j int) {
pq.streams[i], pq.streams[j] = pq.streams[j], pq.streams[i]
}
func (pq *priorityqueue) Push(x interface{}) {
stream := x.(*logproto.Stream)
pq.streams = append(pq.streams, stream)
}
// Pop returns a stream with one entry. It pops the first entry of the first stream
// then re-pushes the remainder of that stream if non-empty back into the queue
func (pq *priorityqueue) Pop() interface{} {
n := pq.Len()
stream := pq.streams[n-1]
pq.streams[n-1] = nil // avoid memory leak
pq.streams = pq.streams[:n-1]
// put the rest of the stream back into the priorityqueue if more entries exist
if len(stream.Entries) > 1 {
remaining := *stream
remaining.Entries = remaining.Entries[1:]
pq.Push(&remaining)
}
stream.Entries = stream.Entries[:1]
return stream
}