Rounds chunk bytes to kb in tsdb and includes benchmarking script (#5479)

* rounds chunk bytes to kb in tsdb and includes benchmarking script

* shellcheck

* fix querier test

pkg/storage/tsdb/index/chunk.go

@@ -10,12 +10,8 @@ type ChunkMeta struct {
 
 	MinTime, MaxTime int64
 
-	// Bytes use an uint64 as an uint32 can only hold [0,4GB)
-	// While this is well within current chunk guidelines (1.5MB being "standard"),
-	// I (owen-d) prefer to overallocate here
-	// Since TSDB accesses are seeked rather than scanned, this choice
-	// should have little effect as long as there is enough memory available
-	Bytes uint64
+	// Bytes stored, rounded to nearest KB
+	KB uint32
 
 	Entries uint32
 }

pkg/storage/tsdb/index/index.go

@@ -458,7 +458,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 		c := chunks[0]
 		w.buf2.PutVarint64(c.MinTime)
 		w.buf2.PutUvarint64(uint64(c.MaxTime - c.MinTime))
-		w.buf2.PutUvarint64(c.Bytes)
+		w.buf2.PutUvarint32(c.KB)
 		w.buf2.PutUvarint32(c.Entries)
 		w.buf2.PutBE32(c.Checksum)
 		t0 := c.MaxTime
@@ -468,7 +468,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
 			// instead of uvarint because chunks may overlap
 			w.buf2.PutVarint64(c.MinTime - t0)
 			w.buf2.PutUvarint64(uint64(c.MaxTime - c.MinTime))
-			w.buf2.PutUvarint64(c.Bytes)
+			w.buf2.PutUvarint32(c.KB)
 			w.buf2.PutUvarint32(c.Entries)
 			t0 = c.MaxTime
 
@@ -1869,7 +1869,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
 
 	t0 := d.Varint64()
 	maxt := int64(d.Uvarint64()) + t0
-	nBytes := d.Uvarint64()
+	kb := uint32(d.Uvarint())
 	entries := uint32(d.Uvarint64())
 	checksum := d.Be32()
 
@@ -1877,7 +1877,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
 		Checksum: checksum,
 		MinTime:  t0,
 		MaxTime:  maxt,
-		Bytes:    nBytes,
+		KB:       kb,
 		Entries:  entries,
 	})
 	t0 = maxt
@@ -1887,7 +1887,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
 		// instead of uvarint because chunks may overlap
 		mint := d.Varint64() + t0
 		maxt := int64(d.Uvarint64()) + mint
-		nBytes := d.Uvarint64()
+		kb := uint32(d.Uvarint())
 		entries := uint32(d.Uvarint64())
 		checksum := d.Be32()
 		t0 = maxt
@@ -1900,7 +1900,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
 			Checksum: checksum,
 			MinTime:  mint,
 			MaxTime:  maxt,
-			Bytes:    nBytes,
+			KB:       kb,
 			Entries:  entries,
 		})
 	}

pkg/storage/tsdb/querier_test.go

@@ -33,14 +33,14 @@ func TestQueryIndex(t *testing.T) {
 					Checksum: 1,
 					MinTime:  1,
 					MaxTime:  10,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 				{
 					Checksum: 2,
 					MinTime:  5,
 					MaxTime:  15,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 			},
@@ -52,14 +52,14 @@ func TestQueryIndex(t *testing.T) {
 					Checksum: 3,
 					MinTime:  20,
 					MaxTime:  30,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 				{
 					Checksum: 4,
 					MinTime:  40,
 					MaxTime:  50,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 			},
@@ -71,14 +71,14 @@ func TestQueryIndex(t *testing.T) {
 					Checksum: 1,
 					MinTime:  1,
 					MaxTime:  10,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 				{
 					Checksum: 2,
 					MinTime:  5,
 					MaxTime:  15,
-					Bytes:    10,
+					KB:       10,
 					Entries:  10,
 				},
 			},

tools/tsdb/tsdb-map/diff.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+old=$1
+new=$2
+
+echo benchmarks:
+echo
+
+benchstat \
+<(LOKI_TSDB_PATH="${old}" go test github.com/grafana/loki/tools/tsdb/tsdb-map -bench=BenchmarkQuery -run '^$' -benchmem) \
+<(LOKI_TSDB_PATH="${new}" go test github.com/grafana/loki/tools/tsdb/tsdb-map -bench=BenchmarkQuery -run '^$' -benchmem)
+
+echo
+echo sizing:
+echo
+
+ls -lh "${old}"
+ls -lh "${new}"

tools/tsdb/tsdb-map/main.go

@@ -84,8 +84,8 @@ func main() {
 				Checksum: extractChecksumFromChunkID(entry.ChunkID),
 				MinTime:  int64(entry.From),
 				MaxTime:  int64(entry.Through),
-				Bytes:    (3 << 20) / 4, // guess: 0.75mb, 1/2 of the max size
-				Entries:  10000,         // guess: 10k entries
+				KB:       ((3 << 20) / 4) / 1024, // guess: 0.75mb, 1/2 of the max size, rounded to KB
+				Entries:  10000,                  // guess: 10k entries
 			}})
 		}