loki/pkg/pattern/drain/chunk.go

package drain

import (
	"sort"
	"time"

	"github.com/prometheus/common/model"

	"github.com/grafana/loki/v3/pkg/logproto"
	"github.com/grafana/loki/v3/pkg/pattern/iter"
)

const (
	TimeResolution = model.Time(int64(time.Second*10) / 1e6)

	defaultVolumeSize = 500

	maxChunkTime = 1 * time.Hour
)

type Chunks []Chunk

type Chunk struct {
	Samples []logproto.PatternSample
}

func newChunk(ts model.Time) Chunk {
	maxSize := int(maxChunkTime.Nanoseconds()/TimeResolution.UnixNano()) + 1
	v := Chunk{Samples: make([]logproto.PatternSample, 1, maxSize)}
	v.Samples[0] = logproto.PatternSample{
		Timestamp: ts,
		Value:     1,
	}
	return v
}

func (c Chunk) spaceFor(ts model.Time) bool {
	if len(c.Samples) == 0 {
		return true
	}

	return ts.Sub(c.Samples[0].Timestamp) < maxChunkTime
}

// ForRange returns samples with only the values
// in the given range [start:end) and aggregates them by step duration.
// start and end are in milliseconds since epoch. step is a duration in milliseconds.
func (c Chunk) ForRange(start, end, step model.Time) []logproto.PatternSample {
	if len(c.Samples) == 0 {
		return nil
	}
	first := c.Samples[0].Timestamp
	last := c.Samples[len(c.Samples)-1].Timestamp
	if start >= end || first >= end || last < start {
		return nil
	}
	var lo int
	if start > first {
		lo = sort.Search(len(c.Samples), func(i int) bool {
			return c.Samples[i].Timestamp >= start
		})
	}
	hi := len(c.Samples)
	if end < last {
		hi = sort.Search(len(c.Samples), func(i int) bool {
			return c.Samples[i].Timestamp >= end
		})
	}

	if c.Samples[lo].Timestamp > c.Samples[hi-1].Timestamp {
		return nil
	}

	if step == TimeResolution {
		return c.Samples[lo:hi]
	}

	// Re-scale samples into step-sized buckets
	currentStep := truncateTimestamp(c.Samples[lo].Timestamp, step)
	aggregatedSamples := make([]logproto.PatternSample, 0, ((c.Samples[hi-1].Timestamp-currentStep)/step)+1)
	aggregatedSamples = append(aggregatedSamples, logproto.PatternSample{
		Timestamp: currentStep,
		Value:     0,
	})
	for _, sample := range c.Samples[lo:hi] {
		if sample.Timestamp >= currentStep+step {
			stepForSample := truncateTimestamp(sample.Timestamp, step)
			for i := currentStep + step; i <= stepForSample; i += step {
				aggregatedSamples = append(aggregatedSamples, logproto.PatternSample{
					Timestamp: i,
					Value:     0,
				})
			}
			currentStep = stepForSample
		}
		aggregatedSamples[len(aggregatedSamples)-1].Value += sample.Value
	}

	return aggregatedSamples
}

func (c *Chunks) Add(ts model.Time) {
	t := truncateTimestamp(ts, TimeResolution)

	if len(*c) == 0 {
		*c = append(*c, newChunk(t))
		return
	}
	last := &(*c)[len(*c)-1]
	if last.Samples[len(last.Samples)-1].Timestamp == t {
		last.Samples[len(last.Samples)-1].Value++
		return
	}
	if !last.spaceFor(t) {
		*c = append(*c, newChunk(t))
		return
	}
	if ts.Before(last.Samples[len(last.Samples)-1].Timestamp) {
		return
	}
	last.Samples = append(last.Samples, logproto.PatternSample{
		Timestamp: t,
		Value:     1,
	})
}

func (c Chunks) Iterator(pattern string, from, through, step model.Time) iter.Iterator {
	iters := make([]iter.Iterator, 0, len(c))
	for _, chunk := range c {
		samples := chunk.ForRange(from, through, step)
		if len(samples) == 0 {
			continue
		}
		iters = append(iters, iter.NewSlice(pattern, samples))
	}
	return iter.NewNonOverlappingIterator(pattern, iters)
}

func (c Chunks) samples() []*logproto.PatternSample {
	// TODO: []*logproto.PatternSample -> []logproto.PatternSample
	//   Or consider AoS to SoA conversion.
	totalSample := 0
	for i := range c {
		totalSample += len(c[i].Samples)
	}
	s := make([]*logproto.PatternSample, 0, totalSample)
	for _, chunk := range c {
		for i := range chunk.Samples {
			s = append(s, &chunk.Samples[i])
		}
	}
	return s
}

func (c *Chunks) merge(samples []*logproto.PatternSample) []logproto.PatternSample {
	toMerge := c.samples()
	// TODO: Avoid allocating a new slice, if possible.
	result := make([]logproto.PatternSample, 0, len(toMerge)+len(samples))
	var i, j int
	for i < len(toMerge) && j < len(samples) {
		if toMerge[i].Timestamp < samples[j].Timestamp {
			result = append(result, *toMerge[i])
			i++
		} else if toMerge[i].Timestamp > samples[j].Timestamp {
			result = append(result, *samples[j])
			j++
		} else {
			result = append(result, logproto.PatternSample{
				Value:     toMerge[i].Value + samples[j].Value,
				Timestamp: toMerge[i].Timestamp,
			})
			i++
			j++
		}
	}
	for ; i < len(toMerge); i++ {
		result = append(result, *toMerge[i])
	}
	for ; j < len(samples); j++ {
		result = append(result, *samples[j])
	}
	*c = Chunks{Chunk{Samples: result}}
	return result
}

func (c *Chunks) prune(olderThan time.Duration) {
	if len(*c) == 0 {
		return
	}
	// go for every chunks, check the last timestamp is after duration from now and remove the chunk
	for i := 0; i < len(*c); i++ {
		if time.Since((*c)[i].Samples[len((*c)[i].Samples)-1].Timestamp.Time()) > olderThan {
			*c = append((*c)[:i], (*c)[i+1:]...)
			i--
		}
	}
}

func (c *Chunks) size() int {
	size := 0
	for _, chunk := range *c {
		for _, sample := range chunk.Samples {
			size += int(sample.Value)
		}
	}
	return size
}

func truncateTimestamp(ts, step model.Time) model.Time { return ts - ts%step }