Reduce test data allocations

chunks/chunk_test.go

@@ -41,8 +41,7 @@ func testChunk(c Chunk) error {
 		ts = int64(1234123324)
 		v  = 1243535.123
 	)
-	i := 0
-	for {
+	for i := 0; i < 300; i++ {
 		ts += int64(rand.Intn(10000) + 1)
 		// v = rand.Float64()
 		if i%2 == 0 {
@@ -68,7 +67,6 @@ func testChunk(c Chunk) error {
 			return err
 		}
 		exp = append(exp, pair{t: ts, v: v})
-		i++
 		// fmt.Println("appended", len(c.Bytes()), c.Bytes())
 	}
 

chunks/xor.go

@@ -16,7 +16,7 @@ type XORChunk struct {
 
 // NewXORChunk returns a new chunk with XOR encoding of the given size.
 func NewXORChunk(size int) *XORChunk {
-	b := make([]byte, 3, 64)
+	b := make([]byte, 3, 128)
 	b[0] = byte(EncXOR)
 
 	return &XORChunk{

cmd/tsdb/main.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
+	"sort"
 	"sync"
 	"time"
 
@@ -172,16 +173,33 @@ func (b *writeBenchmark) ingestScrapesShard(metrics []model.Metric, scrapeCount
 	var sc tsdb.Vector
 	ts := int64(model.Now())
 
+	type sample struct {
+		labels tsdb.Labels
+		value  int64
+	}
+
+	scrape := make(map[uint64]*sample, len(metrics))
+
+	for _, m := range metrics {
+		lset := make(tsdb.Labels, 0, len(m))
+		for k, v := range m {
+			lset = append(lset, tsdb.Label{Name: string(k), Value: string(v)})
+		}
+		sort.Sort(lset)
+
+		scrape[lset.Hash()] = &sample{
+			labels: lset,
+			value:  123456789,
+		}
+	}
+
 	for i := 0; i < scrapeCount; i++ {
 		ts = ts + int64(i*10000)
 		sc.Reset()
 
-		for _, m := range metrics {
-			lset := make([]tsdb.Label, 0, len(m))
-			for k, v := range m {
-				lset = append(lset, tsdb.Label{Name: string(k), Value: string(v)})
-			}
-			sc.Add(lset, float64(rand.Int63()))
+		for _, s := range scrape {
+			s.value += rand.Int63n(1000)
+			sc.Add(s.labels, float64(s.value))
 		}
 		if err := b.storage.ingestScrape(ts, &sc); err != nil {
 			return err

db.go

@@ -70,7 +70,7 @@ func (db *DB) Close() error {
 		fmt.Println("	num samples", shard.head.samples)
 	}
 
-	return fmt.Errorf("not implemented")
+	return nil
 }
 
 // Querier returns a new querier over the database.
@@ -154,7 +154,8 @@ func (ls Labels) Less(i, j int) bool { return ls[i].Name < ls[j].Name }
 
 // Hash returns a hash value for the label set.
 func (ls Labels) Hash() uint64 {
-	b := make([]byte, 0, 512)
+	b := make([]byte, 0, 1024)
+
 	for _, v := range ls {
 		b = append(b, v.Name...)
 		b = append(b, sep)
@@ -222,6 +223,7 @@ func LabelsFromMap(m map[string]string) Labels {
 // Label sets and values must have equal length.
 type Vector struct {
 	Buckets map[uint16][]Sample
+	reused  int
 }
 
 type Sample struct {
@@ -232,7 +234,15 @@ type Sample struct {
 
 // Reset the vector but keep resources allocated.
 func (v *Vector) Reset() {
-	v.Buckets = make(map[uint16][]Sample, len(v.Buckets))
+	// Do a full reset every n-th reusage to avoid memory leaks.
+	if v.Buckets == nil || v.reused > 100 {
+		v.Buckets = make(map[uint16][]Sample, 0)
+		return
+	}
+	for x, bkt := range v.Buckets {
+		v.Buckets[x] = bkt[:0]
+	}
+	v.reused++
 }
 
 // Add a sample to the vector.