Add span events for index stats and result cache (#9552)

**What this PR does / why we need it**:
This PR adds events to the traces to have some extra observability for
how we compute the index stats. We also add some trace events to the
results cache.


![image](https://github.com/grafana/loki/assets/8354290/7566b755-8193-4e46-ba10-37d3377ea31a)


![image](https://github.com/grafana/loki/assets/8354290/d1990150-84b1-4522-9898-6e37c2782c5b)


![image](https://github.com/grafana/loki/assets/8354290/a8c23e7f-a06d-4a47-8cd4-e900fce01e80)


![image](https://github.com/grafana/loki/assets/8354290/d1e15fb6-fb6c-4fe1-9c5f-f1c8164889de)


![image](https://github.com/grafana/loki/assets/8354290/0c0d001e-7083-488c-8809-0446b4b7c852)

**Special notes for your reviewer**:

**Checklist**
- [ ] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [ ] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/upgrading/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e3ec)

pkg/ingester/ingester.go

@@ -1080,6 +1080,9 @@ func (i *Ingester) Series(ctx context.Context, req *logproto.SeriesRequest) (*lo
 }
 
 func (i *Ingester) GetStats(ctx context.Context, req *logproto.IndexStatsRequest) (*logproto.IndexStatsResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "Ingester.GetStats")
+	defer sp.Finish()
+
 	user, err := tenant.TenantID(ctx)
 	if err != nil {
 		return nil, err
@@ -1120,6 +1123,17 @@ func (i *Ingester) GetStats(ctx context.Context, req *logproto.IndexStatsRequest
 	}
 
 	merged := index_stats.MergeStats(resps...)
+	sp.LogKV(
+		"user", user,
+		"from", req.From.Time(),
+		"through", req.Through.Time(),
+		"matchers", syntax.MatchersString(matchers),
+		"streams", merged.Streams,
+		"chunks", merged.Chunks,
+		"bytes", merged.Bytes,
+		"entries", merged.Entries,
+	)
+
 	return &merged, nil
 }
 

pkg/ingester/instance.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log/level"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -560,6 +561,9 @@ func (i *instance) Series(ctx context.Context, req *logproto.SeriesRequest) (*lo
 }
 
 func (i *instance) GetStats(ctx context.Context, req *logproto.IndexStatsRequest) (*logproto.IndexStatsResponse, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "instance.GetStats")
+	defer sp.Finish()
+
 	matchers, err := syntax.ParseMatchers(req.Matchers)
 	if err != nil {
 		return nil, err
@@ -602,6 +606,16 @@ func (i *instance) GetStats(ctx context.Context, req *logproto.IndexStatsRequest
 		return nil, err
 	}
 
+	sp.LogKV(
+		"from", from,
+		"through", through,
+		"matchers", syntax.MatchersString(matchers),
+		"streams", res.Streams,
+		"chunks", res.Chunks,
+		"bytes", res.Bytes,
+		"entries", res.Entries,
+	)
+
 	return res, nil
 }
 

pkg/querier/queryrange/queryrangebase/results_cache.go

@@ -235,6 +235,14 @@ func (s resultsCache) Do(ctx context.Context, r Request) (Response, error) {
 		response Response
 	)
 
+	sp.LogKV(
+		"query", r.GetQuery(),
+		"step", time.UnixMilli(r.GetStep()),
+		"start", time.UnixMilli(r.GetStart()),
+		"end", r.GetEnd(),
+		"key", key,
+	)
+
 	cacheFreshnessCapture := func(id string) time.Duration { return s.limits.MaxCacheFreshness(ctx, id) }
 	maxCacheFreshness := validation.MaxDurationPerTenant(tenantIDs, cacheFreshnessCapture)
 	maxCacheTime := int64(model.Now().Add(-maxCacheFreshness))

pkg/storage/stores/tsdb/index_client.go

@@ -174,6 +174,9 @@ func (c *IndexClient) LabelNamesForMetricName(ctx context.Context, userID string
 }
 
 func (c *IndexClient) Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*stats.Stats, error) {
+	sp, ctx := opentracing.StartSpanFromContext(ctx, "IndexClient.Stats")
+	defer sp.Finish()
+
 	matchers, shard, err := cleanMatchers(matchers...)
 	if err != nil {
 		return nil, err
@@ -220,6 +223,18 @@ func (c *IndexClient) Stats(ctx context.Context, userID string, from, through mo
 	}
 	res := acc.Stats()
 
+	sp.LogKV(
+		"from", from.Time(),
+		"through", through.Time(),
+		"matchers", syntax.MatchersString(matchers),
+		"shard", shard,
+		"intervals", len(intervals),
+		"streams", res.Streams,
+		"chunks", res.Chunks,
+		"bytes", res.Bytes,
+		"entries", res.Entries,
+	)
+
 	return &res, nil
 }