apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add back cache stats for index stats requests (#9816)

Browse Source 
**What this PR does / why we need it**:

In https://github.com/grafana/loki/pull/9536, we added cache stats for
index stats requests. That PR had a bug that inflated the query stats
due to reusing the stats context in the query engine. Therefore, we had
to revert the PR at https://github.com/grafana/loki/pull/9721.

This PR brings back the changes from
https://github.com/grafana/loki/pull/9536 but fixes the inflated starts
by no longer reusing the same context in the query engine, but rather
creating a new one for the shard resolver.

I tested it on a dev cluster and seems to be working fine. here's the
output for the same query:

**Stats with the bug from #9536**:
```
...
Cache.StatsResult.Requests               980
Cache.StatsResult.EntriesRequested       490
Cache.StatsResult.EntriesFound   0
Cache.StatsResult.EntriesStored          490
Cache.StatsResult.BytesSent              0 B
Cache.StatsResult.BytesReceived          0 B
...
Summary.BytesProcessedPerSecond          43 GB
Summary.LinesProcessedPerSecond          93305142
Summary.TotalBytesProcessed              945 GB
Summary.TotalLinesProcessed              2059694183
```

**Stats from _main_**
```
...
Summary.BytesProcessedPerSecond          1.6 GB
Summary.LinesProcessedPerSecond          3403718
Summary.TotalBytesProcessed              95 GB
Summary.TotalLinesProcessed              207971404
```

**Stats with fix in this PR**
```
..
Cache.StatsResult.Requests               132
Cache.StatsResult.EntriesRequested       66
Cache.StatsResult.EntriesFound   0
Cache.StatsResult.EntriesStored          66
Cache.StatsResult.BytesSent              0 B
Cache.StatsResult.BytesReceived          0 B
...
Summary.BytesProcessedPerSecond          4.3 GB
Summary.LinesProcessedPerSecond          9468900
Summary.TotalBytesProcessed              95 GB
Summary.TotalLinesProcessed              207793816
```

As can be seen, with the changes in this PR, the summary stats are no
longer inflated.


**Which issue(s) this PR fixes**:
Fixes https://github.com/grafana/loki/pull/9536

**Special notes for your reviewer**:

I think it's ok to skip reviewing the changes from the commit
cherry-picking the changes from
https://github.com/grafana/loki/pull/9536

**Checklist**
- [ ] Reviewed the
[`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md)
guide (**required**)
- [ ] Documentation added
- [ ] Tests updated
- [ ] `CHANGELOG.md` updated
- [ ] If the change is worth mentioning in the release notes, add
`add-to-release-notes` label
- [ ] Changes that require user attention or interaction to upgrade are
documented in `docs/sources/upgrading/_index.md`
- [ ] For Helm chart changes bump the Helm chart version in
`production/helm/loki/Chart.yaml` and update
`production/helm/loki/CHANGELOG.md` and
`production/helm/loki/README.md`. [Example
PR](d10549e3ec)
pull/9812/head^2
Salva Corts 2 years ago committed by GitHub
parent a24ef7b206
commit 6cc581bd26
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 211 additions and 69 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      pkg/logql/metrics.go
  2. 17
      pkg/logqlmodel/stats/context.go
  3. 191
      pkg/logqlmodel/stats/stats.pb.go
  4. 4
      pkg/logqlmodel/stats/stats.proto
  5. 9
      pkg/querier/queryrange/codec_test.go
  6. 9
      pkg/querier/queryrange/prometheus_test.go
  7. 11
      pkg/querier/queryrange/querysharding.go
  8. 9
      pkg/util/marshal/legacy/marshal_test.go
  9. 27
      pkg/util/marshal/marshal_test.go

3
pkg/logql/metrics.go
Unescape View File

@ -142,6 +142,9 @@ func RecordRangeAndInstantQueryMetrics(
"cache_index_req", stats.Caches.Index.EntriesRequested,
"cache_index_hit", stats.Caches.Index.EntriesFound,
"cache_index_download_time", stats.Caches.Index.CacheDownloadTime(),
"cache_stats_results_req", stats.Caches.StatsResult.EntriesRequested,
"cache_stats_results_hit", stats.Caches.StatsResult.EntriesFound,
"cache_stats_results_download_time", stats.Caches.StatsResult.CacheDownloadTime(),
"cache_result_req", stats.Caches.Result.EntriesRequested,
"cache_result_hit", stats.Caches.Result.EntriesFound,
"cache_result_download_time", stats.Caches.Result.CacheDownloadTime(),

17
pkg/logqlmodel/stats/context.go
Unescape View File

@ -92,9 +92,10 @@ func (c *Context) Ingester() Ingester {
// Caches returns the cache statistics accumulated so far.
func (c *Context) Caches() Caches {
return Caches{
Chunk: c.caches.Chunk,
Index: c.caches.Index,
Result: c.caches.Result,
Chunk: c.caches.Chunk,
Index: c.caches.Index,
Result: c.caches.Result,
StatsResult: c.caches.StatsResult,
}
}
@ -198,6 +199,7 @@ func (c *Caches) Merge(m Caches) {
c.Chunk.Merge(m.Chunk)
c.Index.Merge(m.Index)
c.Result.Merge(m.Result)
c.StatsResult.Merge(m.StatsResult)
}
func (c *Cache) Merge(m Cache) {
@ -395,6 +397,8 @@ func (c *Context) getCacheStatsByType(t CacheType) *Cache {
stats = &c.caches.Index
case ResultCache:
stats = &c.caches.Result
case StatsResultCache:
stats = &c.caches.StatsResult
default:
return nil
}
@ -459,6 +463,13 @@ func (c Caches) Log(log log.Logger) {
"Cache.Index.BytesSent", humanize.Bytes(uint64(c.Index.BytesSent)),
"Cache.Index.BytesReceived", humanize.Bytes(uint64(c.Index.BytesReceived)),
"Cache.Index.DownloadTime", c.Index.CacheDownloadTime(),
"Cache.StatsResult.Requests", c.StatsResult.Requests,
"Cache.StatsResult.EntriesRequested", c.StatsResult.EntriesRequested,
"Cache.StatsResult.EntriesFound", c.StatsResult.EntriesFound,
"Cache.StatsResult.EntriesStored", c.StatsResult.EntriesStored,
"Cache.StatsResult.BytesSent", humanize.Bytes(uint64(c.StatsResult.BytesSent)),
"Cache.StatsResult.BytesReceived", humanize.Bytes(uint64(c.StatsResult.BytesReceived)),
"Cache.Result.DownloadTime", c.Result.CacheDownloadTime(),
"Cache.Result.Requests", c.Result.Requests,
"Cache.Result.EntriesRequested", c.Result.EntriesRequested,
"Cache.Result.EntriesFound", c.Result.EntriesFound,

191
pkg/logqlmodel/stats/stats.pb.go
Unescape View File

@ -95,9 +95,10 @@ func (m *Result) GetCaches() Caches {
}
type Caches struct {
Chunk Cache `protobuf:"bytes,1,opt,name=chunk,proto3" json:"chunk"`
Index Cache `protobuf:"bytes,2,opt,name=index,proto3" json:"index"`
Result Cache `protobuf:"bytes,3,opt,name=result,proto3" json:"result"`
Chunk Cache `protobuf:"bytes,1,opt,name=chunk,proto3" json:"chunk"`
Index Cache `protobuf:"bytes,2,opt,name=index,proto3" json:"index"`
Result Cache `protobuf:"bytes,3,opt,name=result,proto3" json:"result"`
StatsResult Cache `protobuf:"bytes,4,opt,name=statsResult,proto3" json:"statsResult"`
}
func (m *Caches) Reset() { *m = Caches{} }
@ -153,6 +154,13 @@ func (m *Caches) GetResult() Cache {
return Cache{}
}
func (m *Caches) GetStatsResult() Cache {
if m != nil {
return m.StatsResult
}
return Cache{}
}
// Summary is the summary of a query statistics.
type Summary struct {
// Total bytes processed per second.
@ -668,68 +676,69 @@ func init() {
func init() { proto.RegisterFile("pkg/logqlmodel/stats/stats.proto", fileDescriptor_6cdfe5d2aea33ebb) }
var fileDescriptor_6cdfe5d2aea33ebb = []byte{
// 967 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xcf, 0x6f, 0xe3, 0x44,
0x14, 0x8e, 0x9b, 0x3a, 0x69, 0x87, 0xfe, 0xda, 0xd9, 0x2e, 0x6b, 0x40, 0xb2, 0xab, 0x9c, 0x2a,
0x81, 0x1a, 0xf1, 0x43, 0x42, 0x20, 0x56, 0x42, 0xee, 0xb2, 0x52, 0xa5, 0x45, 0x2c, 0xaf, 0x70,
0xe1, 0xe6, 0xd8, 0xb3, 0x89, 0x55, 0xc7, 0x4e, 0x3d, 0x36, 0xec, 0xde, 0xb8, 0x71, 0x84, 0x3f,
0x82, 0x03, 0x17, 0xfe, 0x8f, 0x3d, 0xf6, 0xb8, 0x27, 0x8b, 0xa6, 0x17, 0xe4, 0xd3, 0x4a, 0xdc,
0x11, 0x9a, 0x37, 0x13, 0xdb, 0xe3, 0x38, 0x12, 0x97, 0xcc, 0xbc, 0xef, 0x7b, 0xdf, 0xcc, 0xf3,
0xcc, 0x7b, 0x6f, 0x42, 0x4e, 0x16, 0x57, 0xd3, 0x71, 0x94, 0x4c, 0xaf, 0xa3, 0x79, 0x12, 0xb0,
0x68, 0xcc, 0x33, 0x2f, 0xe3, 0xf2, 0xf7, 0x6c, 0x91, 0x26, 0x59, 0x42, 0x4d, 0x34, 0xde, 0x3d,
0x9e, 0x26, 0xd3, 0x04, 0x91, 0xb1, 0x98, 0x49, 0x72, 0xf4, 0x8f, 0x41, 0x06, 0xc0, 0x78, 0x1e,
0x65, 0xf4, 0x33, 0x32, 0xe4, 0xf9, 0x7c, 0xee, 0xa5, 0x2f, 0x2d, 0xe3, 0xc4, 0x38, 0x7d, 0xeb,
0xa3, 0x83, 0x33, 0xb9, 0xcc, 0xa5, 0x44, 0xdd, 0xc3, 0x57, 0x85, 0xd3, 0x2b, 0x0b, 0x67, 0xe5,
0x06, 0xab, 0x89, 0x90, 0x5e, 0xe7, 0x2c, 0x0d, 0x59, 0x6a, 0x6d, 0x69, 0xd2, 0x6f, 0x25, 0x5a,
0x4b, 0x95, 0x1b, 0xac, 0x26, 0xf4, 0x11, 0xd9, 0x09, 0xe3, 0x29, 0xe3, 0x19, 0x4b, 0xad, 0x3e,
0x6a, 0x0f, 0x95, 0xf6, 0x42, 0xc1, 0xee, 0x91, 0x12, 0x57, 0x8e, 0x50, 0xcd, 0xe8, 0x27, 0x64,
0xe0, 0x7b, 0xfe, 0x8c, 0x71, 0x6b, 0x1b, 0xc5, 0xfb, 0x4a, 0x7c, 0x8e, 0xa0, 0xbb, 0xaf, 0xa4,
0x26, 0x3a, 0x81, 0xf2, 0x1d, 0xfd, 0x6e, 0x90, 0x81, 0xf4, 0xa0, 0x1f, 0x12, 0xd3, 0x9f, 0xe5,
0xf1, 0x95, 0xfa, 0xe6, 0xbd, 0xa6, 0xbe, 0x21, 0x17, 0x2e, 0x20, 0x07, 0x21, 0x09, 0xe3, 0x80,
0xbd, 0x50, 0xdf, 0xba, 0x41, 0x82, 0x2e, 0x20, 0x07, 0x11, 0x66, 0x8a, 0xa7, 0xac, 0xbe, 0x51,
0xd7, 0x1c, 0x28, 0x8d, 0xf2, 0x01, 0x35, 0x8e, 0xca, 0x6d, 0x32, 0x54, 0x87, 0x4f, 0xbf, 0x27,
0x0f, 0x27, 0x2f, 0x33, 0xc6, 0x9f, 0xa5, 0x89, 0xcf, 0x38, 0x67, 0xc1, 0x33, 0x96, 0x5e, 0x32,
0x3f, 0x89, 0x03, 0x8c, 0xbc, 0xef, 0xbe, 0x57, 0x16, 0xce, 0x26, 0x17, 0xd8, 0x44, 0x88, 0x65,
0xa3, 0x30, 0xee, 0x5c, 0x76, 0xab, 0x5e, 0x76, 0x83, 0x0b, 0x6c, 0x22, 0xe8, 0x05, 0xb9, 0x9f,
0x25, 0x99, 0x17, 0xb9, 0xda, 0xb6, 0xf8, 0xf1, 0x7d, 0xf7, 0x61, 0x59, 0x38, 0x5d, 0x34, 0x74,
0x81, 0xd5, 0x52, 0x4f, 0xb5, 0xad, 0xf0, 0xba, 0x9b, 0x4b, 0xe9, 0x34, 0x74, 0x81, 0xf4, 0x94,
0xec, 0xb0, 0x17, 0xcc, 0xff, 0x2e, 0x9c, 0x33, 0xcb, 0x3c, 0x31, 0x4e, 0x0d, 0x77, 0x4f, 0xa4,
0xd5, 0x0a, 0x83, 0x6a, 0x46, 0xdf, 0x27, 0xbb, 0xd7, 0x39, 0xcb, 0x19, 0xba, 0x0e, 0xd0, 0x75,
0xbf, 0x2c, 0x9c, 0x1a, 0x84, 0x7a, 0x4a, 0xcf, 0x08, 0xe1, 0xf9, 0x44, 0x26, 0x34, 0xb7, 0x86,
0x18, 0xd8, 0x41, 0x59, 0x38, 0x0d, 0x14, 0x1a, 0x73, 0xfa, 0x94, 0x1c, 0x63, 0x74, 0x5f, 0xc5,
0x19, 0x72, 0x2c, 0xcb, 0xd3, 0x98, 0x05, 0xd6, 0x0e, 0x2a, 0xad, 0xb2, 0x70, 0x3a, 0x79, 0xe8,
0x44, 0xe9, 0x88, 0x0c, 0xf8, 0x22, 0x0a, 0x33, 0x6e, 0xed, 0xa2, 0x9e, 0x88, 0x44, 0x92, 0x08,
0xa8, 0x11, 0x7d, 0x66, 0x5e, 0x1a, 0x70, 0x8b, 0x34, 0x7c, 0x10, 0x01, 0x35, 0x8e, 0xbe, 0x20,
0x43, 0x55, 0xad, 0x22, 0xc1, 0x79, 0x96, 0xa4, 0xac, 0x55, 0x13, 0x97, 0x02, 0xab, 0x13, 0x1c,
0x5d, 0x40, 0x0e, 0xa3, 0x3f, 0xb7, 0xc8, 0xce, 0x45, 0x5d, 0x94, 0x7b, 0x18, 0x2a, 0x30, 0x91,
0xde, 0x32, 0x41, 0x4d, 0xf7, 0xa8, 0x2c, 0x1c, 0x0d, 0x07, 0xcd, 0xa2, 0x4f, 0x08, 0x45, 0xfb,
0x5c, 0x14, 0x19, 0xff, 0xda, 0xcb, 0x50, 0x2b, 0xb3, 0xf0, 0xed, 0xb2, 0x70, 0x3a, 0x58, 0xe8,
0xc0, 0xaa, 0xdd, 0x5d, 0xb4, 0xb9, 0x4a, 0xba, 0x7a, 0x77, 0x85, 0x83, 0x66, 0xd1, 0xcf, 0xc9,
0x41, 0x9d, 0x32, 0x97, 0x2c, 0xce, 0x54, 0x86, 0xd1, 0xb2, 0x70, 0x5a, 0x0c, 0xb4, 0xec, 0xfa,
0xbc, 0xcc, 0xff, 0x7d, 0x5e, 0xbf, 0x6e, 0x11, 0x13, 0xf9, 0x6a, 0x63, 0xf9, 0x11, 0xc0, 0x9e,
0xab, 0x7a, 0xae, 0x37, 0xae, 0x18, 0x68, 0xd9, 0xf4, 0x1b, 0xf2, 0xa0, 0x81, 0x3c, 0x4e, 0x7e,
0x8a, 0xa3, 0xc4, 0x0b, 0xaa, 0x53, 0x7b, 0xa7, 0x2c, 0x9c, 0x6e, 0x07, 0xe8, 0x86, 0xc5, 0x1d,
0xf8, 0x1a, 0x86, 0x05, 0xd0, 0xaf, 0xef, 0x60, 0x9d, 0x85, 0x0e, 0xac, 0xee, 0xaa, 0xdb, 0x7a,
0xbb, 0x13, 0x58, 0x77, 0x57, 0x1d, 0xfd, 0xd2, 0x27, 0x26, 0xf2, 0xe2, 0x44, 0x66, 0xcc, 0x0b,
0xa4, 0xb3, 0x68, 0x06, 0xcd, 0xab, 0xd0, 0x19, 0x68, 0xd9, 0x9a, 0x16, 0x2f, 0x08, 0xef, 0xa4,
0xad, 0x45, 0x06, 0x5a, 0x36, 0x3d, 0x27, 0xf7, 0x02, 0xe6, 0x27, 0xf3, 0x45, 0x8a, 0xed, 0x42,
0x6e, 0x3d, 0x40, 0xf9, 0x83, 0xb2, 0x70, 0xd6, 0x49, 0x58, 0x87, 0xda, 0x8b, 0xc8, 0x18, 0x86,
0xdd, 0x8b, 0xc8, 0x30, 0xd6, 0x21, 0xfa, 0x88, 0x1c, 0xb6, 0xe3, 0x90, 0xcd, 0xe1, 0x7e, 0x59,
0x38, 0x6d, 0x0a, 0xda, 0x80, 0x90, 0xe3, 0xf5, 0x3e, 0xce, 0x17, 0x51, 0xe8, 0x7b, 0x42, 0xbe,
0x5b, 0xcb, 0x5b, 0x14, 0xb4, 0x81, 0xd1, 0xbf, 0x5b, 0xc4, 0xc4, 0x87, 0x49, 0x94, 0x12, 0x93,
0xed, 0xe6, 0x49, 0x92, 0xc7, 0x5a, 0x21, 0x37, 0x71, 0xd0, 0x2c, 0xfa, 0x25, 0x39, 0x62, 0xab,
0x26, 0x75, 0x9d, 0x8b, 0x96, 0x20, 0x13, 0xd2, 0x74, 0x8f, 0xcb, 0xc2, 0x59, 0xe3, 0x60, 0x0d,
0xa1, 0x9f, 0x92, 0x7d, 0x85, 0x61, 0x8d, 0xc8, 0x87, 0xc3, 0x74, 0xef, 0x95, 0x85, 0xa3, 0x13,
0xa0, 0x9b, 0x42, 0x88, 0x2f, 0x1d, 0x30, 0x9f, 0x85, 0x3f, 0x56, 0xcf, 0x04, 0x0a, 0x35, 0x02,
0x74, 0x53, 0x34, 0x7c, 0x04, 0xb0, 0xf2, 0x65, 0xca, 0x60, 0xc3, 0xaf, 0x40, 0xa8, 0xa7, 0xe2,
0x1d, 0x49, 0x65, 0xac, 0x32, 0x3f, 0x4c, 0xf9, 0x8e, 0xac, 0x30, 0xa8, 0x66, 0xe2, 0x00, 0x83,
0x66, 0x25, 0x0d, 0xeb, 0x5e, 0xd4, 0xc4, 0x41, 0xb3, 0xdc, 0xc9, 0xcd, 0xad, 0xdd, 0x7b, 0x7d,
0x6b, 0xf7, 0xde, 0xdc, 0xda, 0xc6, 0xcf, 0x4b, 0xdb, 0xf8, 0x63, 0x69, 0x1b, 0xaf, 0x96, 0xb6,
0x71, 0xb3, 0xb4, 0x8d, 0xbf, 0x96, 0xb6, 0xf1, 0xf7, 0xd2, 0xee, 0xbd, 0x59, 0xda, 0xc6, 0x6f,
0x77, 0x76, 0xef, 0xe6, 0xce, 0xee, 0xbd, 0xbe, 0xb3, 0x7b, 0x3f, 0x7c, 0x30, 0x0d, 0xb3, 0x59,
0x3e, 0x39, 0xf3, 0x93, 0xf9, 0x78, 0x9a, 0x7a, 0xcf, 0xbd, 0xd8, 0x1b, 0x47, 0xc9, 0x55, 0x38,
0xee, 0xfa, 0x87, 0x38, 0x19, 0xe0, 0xff, 0xbf, 0x8f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x8a,
0x9c, 0x39, 0x00, 0x40, 0x0a, 0x00, 0x00,
// 979 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xcb, 0x6f, 0xe3, 0x44,
0x18, 0x8f, 0x93, 0x75, 0xd2, 0xce, 0xf6, 0xb5, 0xd3, 0x2e, 0x6b, 0x40, 0xb2, 0x2b, 0x9f, 0x2a,
0x81, 0x1a, 0xf1, 0x90, 0x10, 0x88, 0x95, 0x90, 0xbb, 0xac, 0x54, 0x69, 0x11, 0xcb, 0x57, 0xb8,
0x70, 0x73, 0xec, 0xd9, 0xc4, 0xaa, 0x63, 0xa7, 0x7e, 0xc0, 0xee, 0x8d, 0x1b, 0x47, 0xf8, 0x33,
0xb8, 0xf0, 0x7f, 0xec, 0xb1, 0xc7, 0x3d, 0x59, 0x34, 0xbd, 0x80, 0x4f, 0x95, 0xb8, 0x23, 0x34,
0xdf, 0x4c, 0x6c, 0x8f, 0xe3, 0x48, 0x7b, 0x89, 0xe7, 0xfb, 0x3d, 0xe6, 0xfd, 0x7d, 0x13, 0x72,
0xbc, 0xb8, 0x9c, 0x8e, 0xc3, 0x78, 0x7a, 0x15, 0xce, 0x63, 0x9f, 0x85, 0xe3, 0x34, 0x73, 0xb3,
0x54, 0xfc, 0x9e, 0x2e, 0x92, 0x38, 0x8b, 0xa9, 0x8e, 0xc1, 0x7b, 0x47, 0xd3, 0x78, 0x1a, 0x23,
0x32, 0xe6, 0x2d, 0x41, 0xda, 0xff, 0x6a, 0x64, 0x08, 0x2c, 0xcd, 0xc3, 0x8c, 0x7e, 0x4e, 0x46,
0x69, 0x3e, 0x9f, 0xbb, 0xc9, 0x2b, 0x43, 0x3b, 0xd6, 0x4e, 0xee, 0x7f, 0xbc, 0x77, 0x2a, 0xba,
0xb9, 0x10, 0xa8, 0xb3, 0xff, 0xba, 0xb0, 0x7a, 0x65, 0x61, 0xad, 0x64, 0xb0, 0x6a, 0x70, 0xeb,
0x55, 0xce, 0x92, 0x80, 0x25, 0x46, 0x5f, 0xb1, 0x7e, 0x27, 0xd0, 0xda, 0x2a, 0x65, 0xb0, 0x6a,
0xd0, 0xc7, 0x64, 0x2b, 0x88, 0xa6, 0x2c, 0xcd, 0x58, 0x62, 0x0c, 0xd0, 0xbb, 0x2f, 0xbd, 0xe7,
0x12, 0x76, 0x0e, 0xa4, 0xb9, 0x12, 0x42, 0xd5, 0xa2, 0x9f, 0x92, 0xa1, 0xe7, 0x7a, 0x33, 0x96,
0x1a, 0xf7, 0xd0, 0xbc, 0x2b, 0xcd, 0x67, 0x08, 0x3a, 0xbb, 0xd2, 0xaa, 0xa3, 0x08, 0xa4, 0xd6,
0xfe, 0x47, 0x23, 0x43, 0xa1, 0xa0, 0x1f, 0x11, 0xdd, 0x9b, 0xe5, 0xd1, 0xa5, 0x5c, 0xf3, 0x4e,
0xd3, 0xdf, 0xb0, 0x73, 0x09, 0x88, 0x0f, 0xb7, 0x04, 0x91, 0xcf, 0x5e, 0xca, 0xb5, 0x6e, 0xb0,
0xa0, 0x04, 0xc4, 0x87, 0x4f, 0x33, 0xc1, 0x5d, 0x96, 0x6b, 0x54, 0x3d, 0x7b, 0xd2, 0x23, 0x35,
0x20, 0xbf, 0xf4, 0x8c, 0xdc, 0x47, 0x99, 0x38, 0x20, 0xb9, 0x42, 0xd5, 0x7a, 0x28, 0xad, 0x4d,
0x21, 0x34, 0x03, 0xbb, 0xbc, 0x47, 0x46, 0xf2, 0x04, 0xe9, 0x0f, 0xe4, 0xd1, 0xe4, 0x55, 0xc6,
0xd2, 0xe7, 0x49, 0xec, 0xb1, 0x34, 0x65, 0xfe, 0x73, 0x96, 0x5c, 0x30, 0x2f, 0x8e, 0x7c, 0x5c,
0xfe, 0xc0, 0x79, 0xbf, 0x2c, 0xac, 0x4d, 0x12, 0xd8, 0x44, 0xf0, 0x6e, 0xc3, 0x20, 0xea, 0xec,
0xb6, 0x5f, 0x77, 0xbb, 0x41, 0x02, 0x9b, 0x08, 0x7a, 0x4e, 0x0e, 0xb3, 0x38, 0x73, 0x43, 0x47,
0x19, 0x16, 0x77, 0x70, 0xe0, 0x3c, 0x2a, 0x0b, 0xab, 0x8b, 0x86, 0x2e, 0xb0, 0xea, 0xea, 0x99,
0x32, 0x14, 0xee, 0x68, 0xb3, 0x2b, 0x95, 0x86, 0x2e, 0x90, 0x9e, 0x90, 0x2d, 0xf6, 0x92, 0x79,
0xdf, 0x07, 0x73, 0x66, 0xe8, 0xc7, 0xda, 0x89, 0xe6, 0xec, 0xf0, 0xbb, 0xb9, 0xc2, 0xa0, 0x6a,
0xd1, 0x0f, 0xc8, 0xf6, 0x55, 0xce, 0x72, 0x86, 0xd2, 0x21, 0x4a, 0x77, 0xcb, 0xc2, 0xaa, 0x41,
0xa8, 0x9b, 0xf4, 0x94, 0x90, 0x34, 0x9f, 0x88, 0xac, 0x48, 0x8d, 0x11, 0x4e, 0x6c, 0xaf, 0x2c,
0xac, 0x06, 0x0a, 0x8d, 0x36, 0x7d, 0x46, 0x8e, 0x70, 0x76, 0x5f, 0x47, 0x19, 0x72, 0x2c, 0xcb,
0x93, 0x88, 0xf9, 0xc6, 0x16, 0x3a, 0x8d, 0xb2, 0xb0, 0x3a, 0x79, 0xe8, 0x44, 0xa9, 0x4d, 0x86,
0xe9, 0x22, 0x0c, 0xb2, 0xd4, 0xd8, 0x46, 0x3f, 0xe1, 0xb7, 0x51, 0x20, 0x20, 0xbf, 0xa8, 0x99,
0xb9, 0x89, 0x9f, 0x1a, 0xa4, 0xa1, 0x41, 0x04, 0xe4, 0xd7, 0xfe, 0x92, 0x8c, 0x64, 0xca, 0xf3,
0x2c, 0x49, 0xb3, 0x38, 0x61, 0xad, 0xc4, 0xba, 0xe0, 0x58, 0x9d, 0x25, 0x28, 0x01, 0xf1, 0xb1,
0xff, 0xec, 0x93, 0xad, 0xf3, 0x3a, 0xb3, 0x77, 0x70, 0xaa, 0xc0, 0xf8, 0x45, 0x17, 0x17, 0x54,
0x77, 0x0e, 0xca, 0xc2, 0x52, 0x70, 0x50, 0x22, 0xfa, 0x94, 0x50, 0x8c, 0xcf, 0x78, 0xa6, 0xa6,
0xdf, 0xb8, 0x19, 0x7a, 0xc5, 0x2d, 0x7c, 0xa7, 0x2c, 0xac, 0x0e, 0x16, 0x3a, 0xb0, 0x6a, 0x74,
0x07, 0xe3, 0x54, 0x5e, 0xba, 0x7a, 0x74, 0x89, 0x83, 0x12, 0xd1, 0x2f, 0xc8, 0x5e, 0x7d, 0x65,
0x2e, 0x58, 0x94, 0xc9, 0x1b, 0x46, 0xcb, 0xc2, 0x6a, 0x31, 0xd0, 0x8a, 0xeb, 0xfd, 0xd2, 0xdf,
0x7a, 0xbf, 0x7e, 0xeb, 0x13, 0x1d, 0xf9, 0x6a, 0x60, 0xb1, 0x08, 0x60, 0x2f, 0x64, 0x3e, 0xd7,
0x03, 0x57, 0x0c, 0xb4, 0x62, 0xfa, 0x2d, 0x79, 0xd8, 0x40, 0x9e, 0xc4, 0x3f, 0x47, 0x61, 0xec,
0xfa, 0xd5, 0xae, 0xbd, 0x5b, 0x16, 0x56, 0xb7, 0x00, 0xba, 0x61, 0x7e, 0x06, 0x9e, 0x82, 0x61,
0x02, 0x0c, 0xea, 0x33, 0x58, 0x67, 0xa1, 0x03, 0xab, 0x4b, 0x73, 0xab, 0xf0, 0x71, 0xac, 0xbb,
0x34, 0xdb, 0xbf, 0x0e, 0x88, 0x8e, 0x3c, 0xdf, 0x91, 0x19, 0x73, 0x7d, 0x21, 0xe6, 0xc5, 0xa0,
0x79, 0x14, 0x2a, 0x03, 0xad, 0x58, 0xf1, 0xe2, 0x01, 0xe1, 0x99, 0xb4, 0xbd, 0xc8, 0x40, 0x2b,
0xa6, 0x67, 0xe4, 0x81, 0xcf, 0xbc, 0x78, 0xbe, 0x48, 0xb0, 0x5c, 0x88, 0xa1, 0x87, 0x68, 0x7f,
0x58, 0x16, 0xd6, 0x3a, 0x09, 0xeb, 0x50, 0xbb, 0x13, 0x31, 0x87, 0x51, 0x77, 0x27, 0x62, 0x1a,
0xeb, 0x10, 0x7d, 0x4c, 0xf6, 0xdb, 0xf3, 0x10, 0xc5, 0xe1, 0xb0, 0x2c, 0xac, 0x36, 0x05, 0x6d,
0x80, 0xdb, 0xf1, 0x78, 0x9f, 0xe4, 0x8b, 0x30, 0xf0, 0x5c, 0x6e, 0xdf, 0xae, 0xed, 0x2d, 0x0a,
0xda, 0x80, 0xfd, 0x5f, 0x9f, 0xe8, 0xf8, 0x44, 0xf1, 0x54, 0x62, 0xa2, 0xdc, 0x3c, 0x8d, 0xf3,
0x48, 0x49, 0xe4, 0x26, 0x0e, 0x4a, 0x44, 0xbf, 0x22, 0x07, 0x6c, 0x55, 0xa4, 0xae, 0x72, 0x5e,
0x12, 0xc4, 0x85, 0xd4, 0x9d, 0xa3, 0xb2, 0xb0, 0xd6, 0x38, 0x58, 0x43, 0xe8, 0x67, 0x64, 0x57,
0x62, 0x98, 0x23, 0xe2, 0xe1, 0xd0, 0x9d, 0x07, 0x65, 0x61, 0xa9, 0x04, 0xa8, 0x21, 0x37, 0xe2,
0x4b, 0x07, 0xcc, 0x63, 0xc1, 0x4f, 0xd5, 0x33, 0x81, 0x46, 0x85, 0x00, 0x35, 0xe4, 0x05, 0x1f,
0x01, 0xcc, 0x7c, 0x71, 0x65, 0xb0, 0xe0, 0x57, 0x20, 0xd4, 0x4d, 0xfe, 0x8e, 0x24, 0x62, 0xae,
0xe2, 0x7e, 0xe8, 0xe2, 0x1d, 0x59, 0x61, 0x50, 0xb5, 0xf8, 0x06, 0xfa, 0xcd, 0x4c, 0x1a, 0xd5,
0xb5, 0xa8, 0x89, 0x83, 0x12, 0x39, 0x93, 0xeb, 0x1b, 0xb3, 0xf7, 0xe6, 0xc6, 0xec, 0xdd, 0xdd,
0x98, 0xda, 0x2f, 0x4b, 0x53, 0xfb, 0x63, 0x69, 0x6a, 0xaf, 0x97, 0xa6, 0x76, 0xbd, 0x34, 0xb5,
0xbf, 0x96, 0xa6, 0xf6, 0xf7, 0xd2, 0xec, 0xdd, 0x2d, 0x4d, 0xed, 0xf7, 0x5b, 0xb3, 0x77, 0x7d,
0x6b, 0xf6, 0xde, 0xdc, 0x9a, 0xbd, 0x1f, 0x3f, 0x9c, 0x06, 0xd9, 0x2c, 0x9f, 0x9c, 0x7a, 0xf1,
0x7c, 0x3c, 0x4d, 0xdc, 0x17, 0x6e, 0xe4, 0x8e, 0xc3, 0xf8, 0x32, 0x18, 0x77, 0xfd, 0xcd, 0x9c,
0x0c, 0xf1, 0x4f, 0xe4, 0x27, 0xff, 0x07, 0x00, 0x00, 0xff, 0xff, 0x32, 0x2b, 0x75, 0xc7, 0x85,
0x0a, 0x00, 0x00,
}
func (this *Result) Equal(that interface{}) bool {
@ -793,6 +802,9 @@ func (this *Caches) Equal(that interface{}) bool {
if !this.Result.Equal(&that1.Result) {
return false
}
if !this.StatsResult.Equal(&that1.StatsResult) {
return false
}
return true
}
func (this *Summary) Equal(that interface{}) bool {
@ -1037,11 +1049,12 @@ func (this *Caches) GoString() string {
if this == nil {
return "nil"
}
s := make([]string, 0, 7)
s := make([]string, 0, 8)
s = append(s, "&stats.Caches{")
s = append(s, "Chunk: "+strings.Replace(this.Chunk.GoString(), `&`, ``, 1)+",\n")
s = append(s, "Index: "+strings.Replace(this.Index.GoString(), `&`, ``, 1)+",\n")
s = append(s, "Result: "+strings.Replace(this.Result.GoString(), `&`, ``, 1)+",\n")
s = append(s, "StatsResult: "+strings.Replace(this.StatsResult.GoString(), `&`, ``, 1)+",\n")
s = append(s, "}")
return strings.Join(s, "")
}
@ -1223,6 +1236,16 @@ func (m *Caches) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
{
size, err := m.StatsResult.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
return 0, err
}
i -= size
i = encodeVarintStats(dAtA, i, uint64(size))
}
i--
dAtA[i] = 0x22
{
size, err := m.Result.MarshalToSizedBuffer(dAtA[:i])
if err != nil {
@ -1616,6 +1639,8 @@ func (m *Caches) Size() (n int) {
n += 1 + l + sovStats(uint64(l))
l = m.Result.Size()
n += 1 + l + sovStats(uint64(l))
l = m.StatsResult.Size()
n += 1 + l + sovStats(uint64(l))
return n
}
@ -1796,6 +1821,7 @@ func (this *Caches) String() string {
`Chunk:` + strings.Replace(strings.Replace(this.Chunk.String(), "Cache", "Cache", 1), `&`, ``, 1) + `,`,
`Index:` + strings.Replace(strings.Replace(this.Index.String(), "Cache", "Cache", 1), `&`, ``, 1) + `,`,
`Result:` + strings.Replace(strings.Replace(this.Result.String(), "Cache", "Cache", 1), `&`, ``, 1) + `,`,
`StatsResult:` + strings.Replace(strings.Replace(this.StatsResult.String(), "Cache", "Cache", 1), `&`, ``, 1) + `,`,
`}`,
}, "")
return s
@ -2208,6 +2234,39 @@ func (m *Caches) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field StatsResult", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowStats
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthStats
}
postIndex := iNdEx + msglen
if postIndex < 0 {
return ErrInvalidLengthStats
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
if err := m.StatsResult.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipStats(dAtA[iNdEx:])

4
pkg/logqlmodel/stats/stats.proto
Unescape View File

@ -41,6 +41,10 @@ message Caches {
(gogoproto.nullable) = false,
(gogoproto.jsontag) = "result"
];
Cache statsResult = 4 [
(gogoproto.nullable) = false,
(gogoproto.jsontag) = "statsResult"
];
}
// Summary is the summary of a query statistics.

9
pkg/querier/queryrange/codec_test.go
Unescape View File

@ -1080,6 +1080,15 @@ var (
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,

9
pkg/querier/queryrange/prometheus_test.go
Unescape View File

@ -66,6 +66,15 @@ var emptyStats = `"stats": {
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,

11
pkg/querier/queryrange/querysharding.go
Unescape View File

@ -18,6 +18,7 @@ import (
"github.com/grafana/loki/pkg/loghttp"
"github.com/grafana/loki/pkg/logql"
"github.com/grafana/loki/pkg/logqlmodel"
"github.com/grafana/loki/pkg/logqlmodel/stats"
"github.com/grafana/loki/pkg/querier/astmapper"
"github.com/grafana/loki/pkg/querier/queryrange/queryrangebase"
"github.com/grafana/loki/pkg/storage/config"
@ -161,6 +162,13 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que
return nil, err
}
// The shard resolver uses index stats to determine the number of shards.
// We want to store the cache stats for the requests to get the index stats.
// Later on, the query engine overwrites the stats context with other stats,
// so we create a separate stats context here for the resolver that we
// will merge with the stats returned from the engine.
resolverStats, ctx := stats.NewContext(ctx)
resolver, ok := shardResolverForConf(
ctx,
conf,
@ -217,6 +225,9 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que
return nil, err
}
// Merge index stats result cache stats from shard resolver into the query stats.
res.Statistics.Caches.StatsResult.Merge(resolverStats.Caches().StatsResult)
value, err := marshal.NewResultValue(res.Data)
if err != nil {
return nil, err

9
pkg/util/marshal/legacy/marshal_test.go
Unescape View File

@ -97,6 +97,15 @@ var queryTests = []struct {
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,

27
pkg/util/marshal/marshal_test.go
Unescape View File

@ -106,6 +106,15 @@ var queryTests = []struct {
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
@ -243,6 +252,15 @@ var queryTests = []struct {
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,
@ -401,6 +419,15 @@ var queryTests = []struct {
"requests": 0,
"downloadTime": 0
},
"statsResult": {
"entriesFound": 0,
"entriesRequested": 0,
"entriesStored": 0,
"bytesReceived": 0,
"bytesSent": 0,
"requests": 0,
"downloadTime": 0
},
"result": {
"entriesFound": 0,
"entriesRequested": 0,

Write Preview
Loading…
Cancel
Save