Improve wal entries encoding. (#3153)

Little trick to avoid memory allocation with regards to bytes slice and string.

benchmp:

```
❯ benchcmp  before.txt after.txt
benchmark                      old ns/op     new ns/op     delta
Benchmark_EncodeEntries-16     1699362       1055627       -37.88%

benchmark                      old allocs     new allocs     delta
Benchmark_EncodeEntries-16     20025          25             -99.88%

benchmark                      old bytes     new bytes     delta
Benchmark_EncodeEntries-16     5625393       4665376       -17.07%
```

This originated from an investigation on CPU usage of ingester.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
pull/3156/head
Cyril Tovena 5 years ago committed by GitHub
parent cfbc5f84e6
commit 2a90c4ceb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 11
      pkg/ingester/encoding.go
  2. 33
      pkg/ingester/encoding_test.go

@ -113,14 +113,11 @@ outer:
for _, s := range ref.Entries {
buf.PutVarint64(s.Timestamp.UnixNano() - first)
// denote line length
byteLine := []byte(s.Line)
buf.PutUvarint(len(byteLine))
buf.PutBytes(byteLine)
buf.PutUvarint(len(s.Line))
buf.PutString(s.Line)
}
}
return buf.Get()
}
func decodeEntries(b []byte, rec *WALRecord) error {
@ -164,7 +161,6 @@ func decodeEntries(b []byte, rec *WALRecord) error {
return errors.Errorf("unexpected %d bytes left in entry", len(dec.B))
}
return nil
}
func decodeWALRecord(b []byte, walRec *WALRecord) (err error) {
@ -205,7 +201,6 @@ func decodeWALRecord(b []byte, walRec *WALRecord) (err error) {
func EncWith(b []byte) (res Encbuf) {
res.B = b
return res
}
// Encbuf extends encoding.Encbuf with support for multi byte encoding
@ -213,7 +208,7 @@ type Encbuf struct {
encoding.Encbuf
}
func (e *Encbuf) PutBytes(c []byte) { e.B = append(e.B, c...) }
func (e *Encbuf) PutString(s string) { e.B = append(e.B, s...) }
func DecWith(b []byte) (res Decbuf) {
res.B = b

@ -88,6 +88,38 @@ func Test_Encoding_Entries(t *testing.T) {
require.Equal(t, record, decoded)
}
func Benchmark_EncodeEntries(b *testing.B) {
var entries []logproto.Entry
for i := int64(0); i < 10000; i++ {
entries = append(entries, logproto.Entry{
Timestamp: time.Unix(0, i),
Line: fmt.Sprintf("long line with a lot of data like a log %d", i),
})
}
record := &WALRecord{
entryIndexMap: make(map[uint64]int),
UserID: "123",
RefEntries: []RefEntries{
{
Ref: 456,
Entries: entries,
},
{
Ref: 789,
Entries: entries,
},
},
}
b.ReportAllocs()
b.ResetTimer()
buf := recordPool.GetBytes()[:0]
defer recordPool.PutBytes(buf)
for n := 0; n < b.N; n++ {
record.encodeEntries(buf)
}
}
func fillChunk(t *testing.T, c chunkenc.Chunk) int64 {
t.Helper()
var i, inserted int64
@ -115,7 +147,6 @@ func dummyConf() *Config {
}
func Test_EncodingChunks(t *testing.T) {
conf := dummyConf()
c := chunkenc.NewMemChunk(chunkenc.EncGZIP, conf.BlockSize, conf.TargetChunkSize)
fillChunk(t, c)

Loading…
Cancel
Save