Add e2e benchmarks for head and block Queries (#467)

* Add e2e had and block query benchmarks

Signed-off-by: Krasi Georgiev <kgeorgie@redhat.com>

block_test.go

@@ -151,6 +151,18 @@ func TestCorruptedChunk(t *testing.T) {
 
 // createBlock creates a block with given set of series and returns its dir.
 func createBlock(tb testing.TB, dir string, series []Series) string {
+	head := createHead(tb, series)
+	compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil)
+	testutil.Ok(tb, err)
+
+	testutil.Ok(tb, os.MkdirAll(dir, 0777))
+
+	ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime(), nil)
+	testutil.Ok(tb, err)
+	return filepath.Join(dir, ulid.String())
+}
+
+func createHead(tb testing.TB, series []Series) *Head {
 	head, err := NewHead(nil, nil, nil, 2*60*60*1000)
 	testutil.Ok(tb, err)
 	defer head.Close()
@@ -174,15 +186,7 @@ func createBlock(tb testing.TB, dir string, series []Series) string {
 	}
 	err = app.Commit()
 	testutil.Ok(tb, err)
-
-	compactor, err := NewLeveledCompactor(context.Background(), nil, log.NewNopLogger(), []int64{1000000}, nil)
-	testutil.Ok(tb, err)
-
-	testutil.Ok(tb, os.MkdirAll(dir, 0777))
-
-	ulid, err := compactor.Write(dir, head, head.MinTime(), head.MaxTime(), nil)
-	testutil.Ok(tb, err)
-	return filepath.Join(dir, ulid.String())
+	return head
 }
 
 const (
@@ -197,6 +201,7 @@ func genSeries(totalSeries, labelCount int, mint, maxt int64) []Series {
 	}
 
 	series := make([]Series, totalSeries)
+
 	for i := 0; i < totalSeries; i++ {
 		lbls := make(map[string]string, labelCount)
 		lbls[defaultLabelName] = strconv.Itoa(i)

querier_test.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -1225,45 +1226,6 @@ func BenchmarkMergedSeriesSet(b *testing.B) {
 	}
 }
 
-func BenchmarkPersistedQueries(b *testing.B) {
-	for _, nSeries := range []int{10, 100} {
-		for _, nSamples := range []int64{1000, 10000, 100000} {
-			b.Run(fmt.Sprintf("series=%d,samplesPerSeries=%d", nSeries, nSamples), func(b *testing.B) {
-				dir, err := ioutil.TempDir("", "bench_persisted")
-				testutil.Ok(b, err)
-				defer func() {
-					testutil.Ok(b, os.RemoveAll(dir))
-				}()
-
-				block, err := OpenBlock(nil, createBlock(b, dir, genSeries(nSeries, 10, 1, int64(nSamples))), nil)
-				testutil.Ok(b, err)
-				defer block.Close()
-
-				q, err := NewBlockQuerier(block, block.Meta().MinTime, block.Meta().MaxTime)
-				testutil.Ok(b, err)
-				defer q.Close()
-
-				b.ResetTimer()
-				b.ReportAllocs()
-
-				for i := 0; i < b.N; i++ {
-					ss, err := q.Select(labels.NewMustRegexpMatcher("__name__", ".+"))
-					for ss.Next() {
-						s := ss.At()
-						s.Labels()
-						it := s.Iterator()
-						for it.Next() {
-						}
-						testutil.Ok(b, it.Err())
-					}
-					testutil.Ok(b, ss.Err())
-					testutil.Ok(b, err)
-				}
-			})
-		}
-	}
-}
-
 type mockChunkReader map[uint64]chunkenc.Chunk
 
 func (cr mockChunkReader) Chunk(id uint64) (chunkenc.Chunk, error) {
@@ -1584,19 +1546,7 @@ func BenchmarkQueryIterator(b *testing.B) {
 				}
 				defer sq.Close()
 
-				b.ResetTimer()
-				b.ReportAllocs()
-
-				ss, err := sq.Select(labels.NewMustRegexpMatcher("__name__", ".*"))
-				testutil.Ok(b, err)
-				for ss.Next() {
-					it := ss.At().Iterator()
-					for it.Next() {
-					}
-					testutil.Ok(b, it.Err())
-				}
-				testutil.Ok(b, ss.Err())
-				testutil.Ok(b, err)
+				benchQuery(b, c.numSeries, sq, labels.Selector{labels.NewMustRegexpMatcher("__name__", ".*")})
 			})
 		}
 	}
@@ -2156,3 +2106,124 @@ func TestClose(t *testing.T) {
 	testutil.Ok(t, q.Close())
 	testutil.NotOk(t, q.Close())
 }
+
+func BenchmarkQueries(b *testing.B) {
+	cases := map[string]labels.Selector{
+		"Eq Matcher: Expansion - 1": labels.Selector{
+			labels.NewEqualMatcher("la", "va"),
+		},
+		"Eq Matcher: Expansion - 2": labels.Selector{
+			labels.NewEqualMatcher("la", "va"),
+			labels.NewEqualMatcher("lb", "vb"),
+		},
+
+		"Eq Matcher: Expansion - 3": labels.Selector{
+			labels.NewEqualMatcher("la", "va"),
+			labels.NewEqualMatcher("lb", "vb"),
+			labels.NewEqualMatcher("lc", "vc"),
+		},
+		"Regex Matcher: Expansion - 1": labels.Selector{
+			labels.NewMustRegexpMatcher("la", ".*va"),
+		},
+		"Regex Matcher: Expansion - 2": labels.Selector{
+			labels.NewMustRegexpMatcher("la", ".*va"),
+			labels.NewMustRegexpMatcher("lb", ".*vb"),
+		},
+		"Regex Matcher: Expansion - 3": labels.Selector{
+			labels.NewMustRegexpMatcher("la", ".*va"),
+			labels.NewMustRegexpMatcher("lb", ".*vb"),
+			labels.NewMustRegexpMatcher("lc", ".*vc"),
+		},
+	}
+
+	queryTypes := make(map[string]Querier)
+	defer func() {
+		for _, q := range queryTypes {
+			// Can't run a check for error here as some of these will fail as
+			// queryTypes is using the same slice for the different block queriers
+			// and would have been closed in the previous iterration.
+			q.Close()
+		}
+	}()
+
+	for title, selectors := range cases {
+		for _, nSeries := range []int{10} {
+			for _, nSamples := range []int64{1000, 10000, 100000} {
+				dir, err := ioutil.TempDir("", "test_persisted_query")
+				testutil.Ok(b, err)
+				defer func() {
+					testutil.Ok(b, os.RemoveAll(dir))
+				}()
+
+				series := genSeries(nSeries, 5, 1, int64(nSamples))
+
+				// Add some common labels to make the matchers select these series.
+				{
+					var commonLbls labels.Labels
+					for _, selector := range selectors {
+						switch sel := selector.(type) {
+						case *labels.EqualMatcher:
+							commonLbls = append(commonLbls, labels.Label{Name: sel.Name(), Value: sel.Value()})
+						case *labels.RegexpMatcher:
+							commonLbls = append(commonLbls, labels.Label{Name: sel.Name(), Value: sel.Value()})
+						}
+					}
+					for i := range commonLbls {
+						s := series[i].(*mockSeries)
+						allLabels := append(commonLbls, s.Labels()...)
+						s = &mockSeries{
+							labels:   func() labels.Labels { return allLabels },
+							iterator: s.iterator,
+						}
+						series[i] = s
+					}
+				}
+
+				qs := []Querier{}
+				for x := 0; x <= 10; x++ {
+					block, err := OpenBlock(nil, createBlock(b, dir, series), nil)
+					testutil.Ok(b, err)
+					q, err := NewBlockQuerier(block, 1, int64(nSamples))
+					testutil.Ok(b, err)
+					qs = append(qs, q)
+				}
+				queryTypes["_1-Block"] = &querier{blocks: qs[:1]}
+				queryTypes["_3-Blocks"] = &querier{blocks: qs[0:3]}
+				queryTypes["_10-Blocks"] = &querier{blocks: qs}
+
+				head := createHead(b, series)
+				qHead, err := NewBlockQuerier(head, 1, int64(nSamples))
+				testutil.Ok(b, err)
+				queryTypes["_Head"] = qHead
+
+				for qtype, querier := range queryTypes {
+					b.Run(title+qtype+"_nSeries:"+strconv.Itoa(nSeries)+"_nSamples:"+strconv.Itoa(int(nSamples)), func(b *testing.B) {
+						expExpansions, err := strconv.Atoi(string(title[len(title)-1]))
+						testutil.Ok(b, err)
+						benchQuery(b, expExpansions, querier, selectors)
+					})
+				}
+			}
+		}
+	}
+}
+
+func benchQuery(b *testing.B, expExpansions int, q Querier, selectors labels.Selector) {
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		ss, err := q.Select(selectors...)
+		testutil.Ok(b, err)
+		var actualExpansions int
+		for ss.Next() {
+			s := ss.At()
+			s.Labels()
+			it := s.Iterator()
+			for it.Next() {
+			}
+			actualExpansions++
+		}
+		testutil.Equals(b, expExpansions, actualExpansions)
+		testutil.Ok(b, ss.Err())
+	}
+}