diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 2f04bab09c..cf38cf295e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -220,7 +220,9 @@ func (e *Engine) Execute(ctx context.Context, params logql.Params) (logqlmodel.R ) queueTime, _ := ctx.Value(httpreq.QueryQueueTimeHTTPHeader).(time.Duration) - stats := stats.FromContext(ctx).Result(durFull, queueTime, builder.Len()) + statsCtx := stats.FromContext(ctx) + statsCtx.AddQuerierExecTime(durFull) + stats := statsCtx.Result(durFull, queueTime, builder.Len()) md := metadata.FromContext(ctx) span.SetStatus(codes.Ok, "") diff --git a/pkg/iter/entry_iterator.go b/pkg/iter/entry_iterator.go index 0292d1955e..0c55ba0588 100644 --- a/pkg/iter/entry_iterator.go +++ b/pkg/iter/entry_iterator.go @@ -368,7 +368,9 @@ func NewQueryClientIterator(client logproto.Querier_QueryClient, direction logpr func (i *queryClientIterator) Next() bool { ctx := i.client.Context() for i.curr == nil || !i.curr.Next() { + start := time.Now() batch, err := i.client.Recv() + stats.FromContext(ctx).AddIngesterRecvWait(time.Since(start)) if err == io.EOF { return false } else if err != nil { diff --git a/pkg/iter/sample_iterator.go b/pkg/iter/sample_iterator.go index 262093872d..8f32ceada5 100644 --- a/pkg/iter/sample_iterator.go +++ b/pkg/iter/sample_iterator.go @@ -5,6 +5,7 @@ import ( "context" "io" "sync" + "time" "github.com/grafana/loki/v3/pkg/logqlmodel/metadata" @@ -486,7 +487,9 @@ func NewSampleQueryClientIterator(client QuerySampleClient) SampleIterator { func (i *sampleQueryClientIterator) Next() bool { ctx := i.client.Context() for i.curr == nil || !i.curr.Next() { + start := time.Now() batch, err := i.client.Recv() + stats.FromContext(ctx).AddIngesterRecvWait(time.Since(start)) if err == io.EOF { return false } else if err != nil { diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index ec4e01c9a4..d9ff30b1e5 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -287,7 +287,9 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { queueTime, _ := ctx.Value(httpreq.QueryQueueTimeHTTPHeader).(time.Duration) - statResult := statsCtx.Result(time.Since(start), queueTime, q.resultLength(data)) + execTime := time.Since(start) + statsCtx.AddQuerierExecTime(execTime) + statResult := statsCtx.Result(execTime, queueTime, q.resultLength(data)) sp.SetAttributes(tracing.KeyValuesToOTelAttributes(statResult.KVList())...) status, _ := server.ClientHTTPStatusAndError(err) diff --git a/pkg/logql/metrics.go b/pkg/logql/metrics.go index 1649824317..13a3bd8cd6 100644 --- a/pkg/logql/metrics.go +++ b/pkg/logql/metrics.go @@ -169,6 +169,7 @@ func RecordRangeAndInstantQueryMetrics( "total_entries", stats.Summary.TotalEntriesReturned, "store_chunks_download_time", stats.ChunksDownloadTime(), "queue_time", logql_stats.ConvertSecondsToNanoseconds(stats.Summary.QueueTime), + "querier_exec_time", logql_stats.ConvertSecondsToNanoseconds(stats.Querier.QuerierExecTime), "splits", stats.Summary.Splits, "shards", stats.Summary.Shards, "query_referenced_structured_metadata", stats.QueryReferencedStructuredMetadata(), @@ -202,6 +203,8 @@ func RecordRangeAndInstantQueryMetrics( "ingester_chunk_matches", stats.Ingester.GetTotalChunksMatched(), // Total ingester reached for this query. "ingester_requests", stats.Ingester.GetTotalReached(), + // Total time querier spent waiting on ingester gRPC Recv(). + "ingester_recv_wait_time", logql_stats.ConvertSecondsToNanoseconds(stats.Ingester.RecvWaitTime), // Total bytes processed but was already in memory (found in the headchunk). Includes structured metadata bytes. "ingester_chunk_head_bytes", util.HumanizeBytes(uint64(stats.Ingester.Store.Chunk.GetHeadChunkBytes())), // Total bytes of compressed chunks (blocks) processed. diff --git a/pkg/logqlmodel/stats/context.go b/pkg/logqlmodel/stats/context.go index 3041e62c8d..65594ac936 100644 --- a/pkg/logqlmodel/stats/context.go +++ b/pkg/logqlmodel/stats/context.go @@ -50,6 +50,17 @@ type Context struct { store Store // result accumulates results for JoinResult. result Result + // recvWaitTime accumulates the time the querier spent waiting on ingester + // gRPC Recv() in nanoseconds. + // We use an int64 nanoseconds value here because the Ingester RecvWaitTime field + // is a double (so that it will pass correctly in json), but there are no + // atomic operations on doubles. + recvWaitTime int64 + // querierExecTime accumulates the querier execution time in nanoseconds + // We use an int64 nanoseconds value here because the Queriers exec time field + // is a double (so that it will pass correctly in json), but there are no + // atomic operations on doubles. + querierExecTime int64 mtx sync.Mutex } @@ -95,6 +106,7 @@ func (c *Context) Ingester() Ingester { TotalBatches: c.ingester.TotalBatches, TotalLinesSent: c.ingester.TotalLinesSent, Store: c.store, + RecvWaitTime: time.Duration(c.recvWaitTime).Seconds(), } } @@ -140,6 +152,8 @@ func (c *Context) Reset() { c.result.Reset() c.caches.Reset() c.index.Reset() + c.recvWaitTime = 0 + c.querierExecTime = 0 } // Result calculates the summary based on store and ingester data. @@ -147,12 +161,23 @@ func (c *Context) Result(execTime time.Duration, queueTime time.Duration, totalE r := c.result r.Merge(Result{ + // ewelch: I'm not sure why we have a separate store object in the context and we don't use the + // store object in the querier object. I didn't try to solve this when adding the querier exec time + // but I suspect this could be simplified? Querier: Querier{ - Store: c.store, + Store: c.store, + QuerierExecTime: time.Duration(c.querierExecTime).Seconds(), }, - Ingester: c.ingester, - Caches: c.caches, - Index: c.index, + Ingester: Ingester{ + TotalReached: c.ingester.TotalReached, + TotalChunksMatched: c.ingester.TotalChunksMatched, + TotalBatches: c.ingester.TotalBatches, + TotalLinesSent: c.ingester.TotalLinesSent, + Store: c.ingester.Store, + RecvWaitTime: time.Duration(c.recvWaitTime).Seconds(), + }, + Caches: c.caches, + Index: c.index, }) r.ComputeSummary(execTime, queueTime, totalEntriesReturned) @@ -253,6 +278,7 @@ func (s *Summary) Merge(m Summary) { func (q *Querier) Merge(m Querier) { q.Store.Merge(m.Store) + q.QuerierExecTime += m.QuerierExecTime } func (i *Ingester) Merge(m Ingester) { @@ -261,6 +287,7 @@ func (i *Ingester) Merge(m Ingester) { i.TotalLinesSent += m.TotalLinesSent i.TotalChunksMatched += m.TotalChunksMatched i.TotalReached += m.TotalReached + i.RecvWaitTime += m.RecvWaitTime } func (i *Index) Merge(m Index) { @@ -316,7 +343,8 @@ func (r *Result) Merge(m Result) { r.Summary.Merge(m.Summary) r.Index.Merge(m.Index) r.ComputeSummary(ConvertSecondsToNanoseconds(r.Summary.ExecTime+m.Summary.ExecTime), - ConvertSecondsToNanoseconds(r.Summary.QueueTime+m.Summary.QueueTime), int(r.Summary.TotalEntriesReturned)) + ConvertSecondsToNanoseconds(r.Summary.QueueTime+m.Summary.QueueTime), + int(r.Summary.TotalEntriesReturned)) } // ConvertSecondsToNanoseconds converts time.Duration representation of seconds (float64) @@ -382,6 +410,12 @@ func (c *Context) AddIngesterReached(i int32) { atomic.AddInt32(&c.ingester.TotalReached, i) } +// AddIngesterRecvWait accumulates the time the querier spent waiting +// for batches from ingesters over gRPC Recv(). +func (c *Context) AddIngesterRecvWait(d time.Duration) { + atomic.AddInt64(&c.recvWaitTime, int64(d)) +} + func (c *Context) AddHeadChunkLines(i int64) { atomic.AddInt64(&c.store.Chunk.HeadChunkLines, i) } @@ -594,6 +628,11 @@ func (c *Context) SetQueryUsedV2Engine() { c.store.QueryUsedV2Engine = true } +// AddQuerierExecTime accumulates the querier execution time. +func (c *Context) AddQuerierExecTime(d time.Duration) { + atomic.AddInt64(&c.querierExecTime, int64(d)) +} + func (c *Context) getCacheStatsByType(t CacheType) *Cache { var stats *Cache switch t { @@ -629,6 +668,7 @@ func (r Result) KVList() []any { "Ingester.TotalChunksMatched", r.Ingester.TotalChunksMatched, "Ingester.TotalBatches", r.Ingester.TotalBatches, "Ingester.TotalLinesSent", r.Ingester.TotalLinesSent, + "Ingester.RecvWaitTime", ConvertSecondsToNanoseconds(r.Ingester.RecvWaitTime), "Ingester.TotalChunksRef", r.Ingester.Store.TotalChunksRef, "Ingester.TotalChunksDownloaded", r.Ingester.Store.TotalChunksDownloaded, "Ingester.ChunksDownloadTime", time.Duration(r.Ingester.Store.ChunksDownloadTime), diff --git a/pkg/logqlmodel/stats/stats.pb.go b/pkg/logqlmodel/stats/stats.pb.go index 5a3adb4b12..3f9915e642 100644 --- a/pkg/logqlmodel/stats/stats.pb.go +++ b/pkg/logqlmodel/stats/stats.pb.go @@ -435,6 +435,10 @@ func (m *Index) GetTotalStreams() int64 { type Querier struct { Store Store `protobuf:"bytes,1,opt,name=store,proto3" json:"store"` + // Querier execution time in seconds. + // Total time the querier spent executing the query. This is separate from execTime which tracks overall execution time. + // Grafana expects time values to be returned in seconds as float. + QuerierExecTime float64 `protobuf:"fixed64,2,opt,name=querierExecTime,proto3" json:"querierExecTime"` } func (m *Querier) Reset() { *m = Querier{} } @@ -476,6 +480,13 @@ func (m *Querier) GetStore() Store { return Store{} } +func (m *Querier) GetQuerierExecTime() float64 { + if m != nil { + return m.QuerierExecTime + } + return 0 +} + type Ingester struct { // Total ingester reached for this query. TotalReached int32 `protobuf:"varint,1,opt,name=totalReached,proto3" json:"totalReached"` @@ -486,6 +497,10 @@ type Ingester struct { // Total lines sent by ingesters. TotalLinesSent int64 `protobuf:"varint,4,opt,name=totalLinesSent,proto3" json:"totalLinesSent"` Store Store `protobuf:"bytes,5,opt,name=store,proto3" json:"store"` + // Total time querier spent waiting on ingester gRPC Recv(), in seconds. + // Omitted from JSON when zero for backwards compatibility. + // Grafana expects time values to be returned in seconds as float + RecvWaitTime float64 `protobuf:"fixed64,6,opt,name=recvWaitTime,proto3" json:"recvWaitTime,omitempty"` } func (m *Ingester) Reset() { *m = Ingester{} } @@ -555,6 +570,13 @@ func (m *Ingester) GetStore() Store { return Store{} } +func (m *Ingester) GetRecvWaitTime() float64 { + if m != nil { + return m.RecvWaitTime + } + return 0 +} + type Store struct { // The total of chunk reference fetched from index. TotalChunksRef int64 `protobuf:"varint,1,opt,name=totalChunksRef,proto3" json:"totalChunksRef"` @@ -1061,113 +1083,116 @@ func init() { func init() { proto.RegisterFile("pkg/logqlmodel/stats/stats.proto", fileDescriptor_6cdfe5d2aea33ebb) } var fileDescriptor_6cdfe5d2aea33ebb = []byte{ - // 1692 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x58, 0xbb, 0x6f, 0xdc, 0x46, - 0x1a, 0xd7, 0x6a, 0xcd, 0x95, 0x3c, 0x7a, 0xd9, 0x23, 0xf9, 0xbc, 0x3e, 0xfb, 0x96, 0xf2, 0x9e, - 0x8d, 0xf3, 0xe1, 0x00, 0x2d, 0xfc, 0x00, 0x0e, 0x77, 0x88, 0x81, 0x84, 0x92, 0x05, 0x18, 0x90, - 0x11, 0xe5, 0x53, 0x9c, 0x04, 0x49, 0x45, 0x91, 0xa3, 0x15, 0x6d, 0x2e, 0xb9, 0x22, 0x87, 0xb2, - 0x05, 0x04, 0x48, 0xfe, 0x84, 0xf4, 0x41, 0xda, 0x20, 0x4d, 0x80, 0x00, 0xa9, 0x52, 0xa7, 0x71, - 0xe9, 0xd2, 0x15, 0x11, 0xcb, 0x4d, 0xc0, 0xca, 0x48, 0x9d, 0x22, 0x98, 0xc7, 0x92, 0x33, 0x24, - 0x77, 0xb5, 0x6e, 0xc4, 0xf9, 0x7e, 0xdf, 0x63, 0x46, 0xdf, 0x7c, 0xaf, 0x59, 0xb4, 0x3e, 0x7c, - 0xda, 0xef, 0xf9, 0x61, 0xff, 0xc8, 0x1f, 0x84, 0x2e, 0xf1, 0x7b, 0x31, 0xb5, 0x69, 0x2c, 0xfe, - 0x6e, 0x0c, 0xa3, 0x90, 0x86, 0xd8, 0xe0, 0xc4, 0xdf, 0xd7, 0xfa, 0x61, 0x3f, 0xe4, 0x48, 0x8f, - 0xad, 0x04, 0xb3, 0xfb, 0xfd, 0x2c, 0x6a, 0x01, 0x89, 0x13, 0x9f, 0xe2, 0xff, 0xa1, 0xb9, 0x38, - 0x19, 0x0c, 0xec, 0xe8, 0xa4, 0xdd, 0x58, 0x6f, 0xdc, 0x5a, 0xb8, 0xb3, 0xbc, 0x21, 0xcc, 0xec, - 0x09, 0xd4, 0x5a, 0x79, 0x91, 0x9a, 0x33, 0x59, 0x6a, 0x8e, 0xc4, 0x60, 0xb4, 0x60, 0xaa, 0x47, - 0x09, 0x89, 0x3c, 0x12, 0xb5, 0x67, 0x35, 0xd5, 0x8f, 0x04, 0x5a, 0xa8, 0x4a, 0x31, 0x18, 0x2d, - 0xf0, 0x7d, 0x34, 0xef, 0x05, 0x7d, 0x12, 0x53, 0x12, 0xb5, 0x9b, 0x5c, 0x77, 0x45, 0xea, 0x3e, - 0x94, 0xb0, 0x75, 0x41, 0x2a, 0xe7, 0x82, 0x90, 0xaf, 0xf0, 0x3d, 0xd4, 0x72, 0x6c, 0xe7, 0x90, - 0xc4, 0xed, 0x73, 0x5c, 0x79, 0x49, 0x2a, 0x6f, 0x72, 0xd0, 0x5a, 0x92, 0xaa, 0x06, 0x17, 0x02, - 0x29, 0x8b, 0x6f, 0x23, 0xc3, 0x0b, 0x5c, 0xf2, 0xbc, 0x6d, 0x70, 0xa5, 0xc5, 0x7c, 0x47, 0x97, - 0x3c, 0x2f, 0x74, 0xb8, 0x08, 0x88, 0x4f, 0xf7, 0xdb, 0x73, 0xa8, 0xb5, 0x99, 0x6b, 0x3b, 0x87, - 0x49, 0xf0, 0x54, 0xba, 0x69, 0x51, 0xdd, 0x52, 0xd9, 0x91, 0x89, 0x80, 0xf8, 0x14, 0x1b, 0xce, - 0x4e, 0x52, 0x51, 0x37, 0x64, 0xff, 0x59, 0xc4, 0x2f, 0x46, 0xba, 0x45, 0xd7, 0x59, 0x96, 0x3a, - 0x52, 0x06, 0xe4, 0x17, 0x6f, 0xa2, 0x05, 0x2e, 0x26, 0xee, 0x54, 0x3a, 0x45, 0x57, 0x5d, 0x95, - 0xaa, 0xaa, 0x20, 0xa8, 0x04, 0xde, 0x46, 0x8b, 0xc7, 0xa1, 0x9f, 0x0c, 0x88, 0xb4, 0x62, 0xd4, - 0x58, 0x59, 0x93, 0x56, 0x34, 0x49, 0xd0, 0x28, 0x66, 0x27, 0x66, 0xb7, 0x3c, 0x3a, 0x4d, 0x6b, - 0x92, 0x1d, 0x55, 0x12, 0x34, 0x8a, 0xfd, 0x53, 0xbe, 0xbd, 0x4f, 0x7c, 0x69, 0x66, 0x6e, 0xd2, - 0x3f, 0xa5, 0x08, 0x82, 0x4a, 0xe0, 0x2f, 0xd0, 0xaa, 0x17, 0xc4, 0xd4, 0x0e, 0xe8, 0x23, 0x42, - 0x23, 0xcf, 0x91, 0xc6, 0xe6, 0x6b, 0x8c, 0x5d, 0x95, 0xc6, 0xea, 0x14, 0xa0, 0x0e, 0xec, 0xfe, - 0xd2, 0x42, 0x73, 0x32, 0x4d, 0xf0, 0x63, 0x74, 0x79, 0xff, 0x84, 0x92, 0x78, 0x37, 0x0a, 0x1d, - 0x12, 0xc7, 0xc4, 0xdd, 0x25, 0xd1, 0x1e, 0x71, 0xc2, 0xc0, 0xe5, 0x01, 0xd3, 0xb4, 0xae, 0x66, - 0xa9, 0x39, 0x4e, 0x04, 0xc6, 0x31, 0x98, 0x59, 0xdf, 0x0b, 0x6a, 0xcd, 0xce, 0x16, 0x66, 0xc7, - 0x88, 0xc0, 0x38, 0x06, 0x7e, 0x88, 0x56, 0x69, 0x48, 0x6d, 0xdf, 0xd2, 0xb6, 0xe5, 0x31, 0xd7, - 0xb4, 0x2e, 0x33, 0x27, 0xd4, 0xb0, 0xa1, 0x0e, 0xcc, 0x4d, 0xed, 0x68, 0x5b, 0xf1, 0x18, 0x54, - 0x4d, 0xe9, 0x6c, 0xa8, 0x03, 0xf1, 0x2d, 0x34, 0x4f, 0x9e, 0x13, 0xe7, 0x63, 0x6f, 0x40, 0x78, - 0xf4, 0x35, 0xac, 0x45, 0x56, 0x00, 0x46, 0x18, 0xe4, 0x2b, 0xfc, 0x1f, 0x74, 0xfe, 0x28, 0x21, - 0x09, 0xe1, 0xa2, 0x2d, 0x2e, 0xba, 0x94, 0xa5, 0x66, 0x01, 0x42, 0xb1, 0xc4, 0x1b, 0x08, 0xc5, - 0xc9, 0xbe, 0x28, 0x3d, 0x31, 0x8f, 0xa3, 0xa6, 0xb5, 0x9c, 0xa5, 0xa6, 0x82, 0x82, 0xb2, 0xc6, - 0x3b, 0x68, 0x8d, 0x9f, 0xee, 0x41, 0x40, 0x45, 0x38, 0xd2, 0x24, 0x0a, 0x88, 0xcb, 0x83, 0xa6, - 0x69, 0xb5, 0xb3, 0xd4, 0xac, 0xe5, 0x43, 0x2d, 0x8a, 0xbb, 0xa8, 0x15, 0x0f, 0x7d, 0x8f, 0xc6, - 0xed, 0xf3, 0x5c, 0x1f, 0xb1, 0xfc, 0x15, 0x08, 0xc8, 0x2f, 0x97, 0x39, 0xb4, 0x23, 0x37, 0x6e, - 0x23, 0x45, 0x86, 0x23, 0x20, 0xbf, 0xf9, 0xa9, 0x76, 0xc3, 0x98, 0x6e, 0x7b, 0x3e, 0x25, 0x11, - 0xf7, 0x5e, 0x7b, 0xa1, 0x74, 0xaa, 0x12, 0x1f, 0x6a, 0x51, 0xfc, 0x15, 0xba, 0xc9, 0xf1, 0x3d, - 0x1a, 0x25, 0x0e, 0x4d, 0x22, 0xe2, 0x3e, 0x22, 0xd4, 0x76, 0x6d, 0x6a, 0x97, 0x42, 0x62, 0x91, - 0x9b, 0xff, 0x77, 0x96, 0x9a, 0xd3, 0x29, 0xc0, 0x74, 0x62, 0xdd, 0x9f, 0x66, 0x91, 0xc1, 0x2b, - 0x2f, 0xbe, 0x8d, 0x16, 0xb8, 0xca, 0x26, 0xab, 0x99, 0xb1, 0xcc, 0x96, 0x15, 0x96, 0xd5, 0x0a, - 0x0c, 0x2a, 0x81, 0xdf, 0x47, 0x17, 0x86, 0xf9, 0x3f, 0x24, 0xf5, 0x44, 0x3a, 0xac, 0x65, 0xa9, - 0x59, 0xe1, 0x41, 0x05, 0xc1, 0xff, 0x47, 0xcb, 0xc2, 0xaf, 0x5b, 0x49, 0x64, 0x53, 0x2f, 0x0c, - 0x64, 0xec, 0xe3, 0x2c, 0x35, 0x4b, 0x1c, 0x28, 0xd1, 0x6c, 0xf7, 0x24, 0x26, 0xae, 0xe5, 0x87, - 0xe1, 0x40, 0x18, 0x15, 0x7d, 0x68, 0x5e, 0xec, 0x5e, 0xe6, 0x41, 0x05, 0xc1, 0xf7, 0xd0, 0xe2, - 0xc8, 0x4b, 0xc4, 0x1e, 0xc4, 0x3c, 0xd8, 0x9b, 0xd6, 0x05, 0x56, 0x10, 0x55, 0x1c, 0x34, 0xaa, - 0xfb, 0x1e, 0x9a, 0x93, 0x9d, 0x95, 0x75, 0x96, 0x98, 0x86, 0x11, 0x29, 0x35, 0xa3, 0x3d, 0x86, - 0x15, 0x9d, 0x85, 0x8b, 0x80, 0xf8, 0x74, 0x7f, 0x9c, 0x45, 0xf3, 0x0f, 0x8b, 0x06, 0x2a, 0x4c, - 0x03, 0x61, 0xa5, 0x4f, 0x94, 0x28, 0x43, 0x39, 0x80, 0xc4, 0x41, 0xa3, 0xf0, 0x36, 0xc2, 0xca, - 0x2d, 0x3c, 0xb2, 0x29, 0xd7, 0x15, 0x8e, 0xff, 0x5b, 0x96, 0x9a, 0x35, 0x5c, 0xa8, 0xc1, 0xf2, - 0xdd, 0x2d, 0x4e, 0xc7, 0xd2, 0xf5, 0xc5, 0xee, 0x12, 0x07, 0x8d, 0x62, 0x57, 0x56, 0x14, 0x8d, - 0x3d, 0x12, 0x50, 0x59, 0x63, 0xf8, 0x95, 0xe9, 0x1c, 0x28, 0xd1, 0x85, 0xbf, 0x8c, 0xa9, 0xfd, - 0xf5, 0xa7, 0x81, 0x0c, 0xce, 0xcf, 0x37, 0x96, 0xc1, 0x44, 0x0e, 0x64, 0x8c, 0x16, 0x1b, 0xe7, - 0x1c, 0x28, 0xd1, 0xf8, 0x43, 0x74, 0x49, 0x41, 0xb6, 0xc2, 0x67, 0x81, 0x1f, 0xda, 0x6e, 0xee, - 0xb5, 0x2b, 0x59, 0x6a, 0xd6, 0x0b, 0x40, 0x3d, 0xcc, 0xee, 0xc0, 0xd1, 0x30, 0x5e, 0x02, 0x9b, - 0xc5, 0x1d, 0x54, 0xb9, 0x50, 0x83, 0x61, 0x07, 0x5d, 0x61, 0xf5, 0xee, 0x04, 0xc8, 0x01, 0x89, - 0x48, 0xe0, 0x10, 0xb7, 0x48, 0xd9, 0xf6, 0x12, 0x8f, 0xe6, 0x9b, 0x59, 0x6a, 0x5e, 0x1f, 0x2b, - 0x34, 0xca, 0x6b, 0x18, 0x6f, 0x07, 0x6f, 0xa2, 0x8b, 0x9c, 0xf9, 0x38, 0x26, 0xee, 0x27, 0x77, - 0x1e, 0x04, 0x7d, 0x2f, 0x20, 0xed, 0x65, 0x6e, 0xfc, 0x52, 0x96, 0x9a, 0x55, 0x26, 0x54, 0xa1, - 0x62, 0xf0, 0x2a, 0x8d, 0x35, 0x0c, 0x1b, 0x33, 0x78, 0x8d, 0x9c, 0x04, 0xe4, 0x20, 0xde, 0x26, - 0xd4, 0x39, 0xcc, 0x5b, 0x8a, 0xea, 0x24, 0x8d, 0x0b, 0x35, 0x18, 0xfe, 0x0c, 0xb5, 0x9d, 0x90, - 0xe7, 0x8c, 0x17, 0x06, 0x9b, 0x61, 0x40, 0xa3, 0xd0, 0xdf, 0xb1, 0x29, 0x09, 0x9c, 0x13, 0xde, - 0x75, 0x9a, 0xd6, 0xb5, 0x2c, 0x35, 0xc7, 0xca, 0xc0, 0x58, 0x0e, 0x76, 0xd1, 0xb5, 0xa1, 0x37, - 0x24, 0xac, 0x3f, 0x7f, 0x1a, 0xd9, 0xc3, 0x21, 0x89, 0x44, 0x6d, 0x20, 0xae, 0xa8, 0xea, 0xa2, - 0x4b, 0xad, 0x67, 0xa9, 0x39, 0x51, 0x0e, 0x26, 0x72, 0xd9, 0x84, 0xce, 0xae, 0x28, 0xdc, 0x7f, - 0x22, 0x27, 0x9e, 0xd1, 0x84, 0xbe, 0x25, 0xd0, 0x62, 0x42, 0x97, 0x62, 0x30, 0x5a, 0x74, 0xff, - 0x98, 0x47, 0x73, 0x52, 0x8a, 0x1f, 0x36, 0x22, 0xbb, 0x11, 0x71, 0x3d, 0xc7, 0xa6, 0x64, 0x8b, - 0x38, 0xe1, 0x60, 0x18, 0x89, 0x72, 0x1f, 0x3e, 0x1b, 0x95, 0x6c, 0x71, 0xd8, 0x09, 0x72, 0x30, - 0x91, 0x8b, 0xfb, 0xe8, 0x1f, 0xe3, 0xf8, 0xbc, 0x77, 0xc8, 0x94, 0xb9, 0x9e, 0xa5, 0xe6, 0x64, - 0x41, 0x98, 0xcc, 0xc6, 0xdf, 0x35, 0x50, 0x6f, 0x9c, 0xc4, 0x98, 0xbe, 0x25, 0x13, 0xec, 0x6e, - 0x96, 0x9a, 0xef, 0xaa, 0x0a, 0xef, 0xaa, 0xc0, 0xb2, 0x86, 0xf5, 0xab, 0x5c, 0x87, 0xfb, 0x58, - 0xd4, 0x3a, 0x9e, 0x35, 0x15, 0x26, 0x54, 0x21, 0xfc, 0x04, 0x75, 0x34, 0xb0, 0xea, 0x4e, 0x91, - 0x0e, 0xdd, 0x2c, 0x35, 0xcf, 0x90, 0x84, 0x33, 0xf8, 0xf8, 0x4b, 0x74, 0x43, 0x93, 0x18, 0xe7, - 0x44, 0x91, 0x32, 0xb7, 0xb2, 0xd4, 0x9c, 0x4a, 0x1e, 0xa6, 0x92, 0x62, 0xe5, 0xb9, 0x68, 0xef, - 0xdc, 0x57, 0x73, 0x45, 0x79, 0xd6, 0x39, 0x50, 0xa2, 0x59, 0x27, 0x1a, 0xda, 0x7d, 0x12, 0xef, - 0x39, 0x76, 0x50, 0x8c, 0x78, 0xbc, 0x13, 0xa9, 0x38, 0x68, 0x14, 0xbe, 0x8f, 0x56, 0x38, 0xad, - 0x94, 0x73, 0x31, 0xdb, 0xad, 0x66, 0xa9, 0x59, 0x66, 0x41, 0x19, 0x60, 0x93, 0x5c, 0x09, 0x12, - 0xee, 0x41, 0xc5, 0x24, 0x57, 0xc7, 0x87, 0x5a, 0x94, 0x8d, 0x4f, 0x0c, 0x1f, 0xf5, 0xd2, 0x85, - 0x62, 0x7c, 0x52, 0x60, 0x50, 0x89, 0xbc, 0x8f, 0x33, 0x17, 0x7c, 0x70, 0x6c, 0x7b, 0xbe, 0xbd, - 0xef, 0x13, 0x39, 0xe9, 0x15, 0x7d, 0x5c, 0xe3, 0x42, 0x0d, 0x96, 0x37, 0xb7, 0x5d, 0xbb, 0x4f, - 0xb4, 0x76, 0xb4, 0x54, 0x6a, 0x6e, 0x65, 0x01, 0xa8, 0x87, 0xbb, 0x3f, 0x1b, 0xc8, 0xe0, 0x75, - 0x9d, 0x5d, 0xea, 0x21, 0xb1, 0x5d, 0x51, 0xe4, 0xb9, 0x77, 0x94, 0x66, 0xaf, 0x73, 0xa0, 0x44, - 0x6b, 0xba, 0xa2, 0x9a, 0x1a, 0x35, 0xba, 0xa2, 0x7e, 0x96, 0x68, 0x96, 0x7b, 0x6e, 0x25, 0x53, - 0x5a, 0x45, 0xee, 0x55, 0x98, 0x50, 0x85, 0xca, 0x46, 0xd4, 0x8a, 0x5e, 0x31, 0x22, 0x8e, 0x51, - 0x85, 0x58, 0x90, 0x95, 0xcf, 0x31, 0x5f, 0x04, 0x59, 0xf9, 0x14, 0x65, 0x80, 0xa9, 0x73, 0x1f, - 0x6f, 0x25, 0x43, 0x9f, 0xa7, 0x4f, 0xac, 0xc6, 0x68, 0x89, 0x05, 0x65, 0x80, 0x87, 0x78, 0xe9, - 0xa1, 0x81, 0x94, 0x10, 0x2f, 0xbd, 0x31, 0xca, 0x00, 0x1e, 0xa2, 0xf5, 0xdc, 0xb1, 0xe3, 0xaa, - 0x81, 0x88, 0xd4, 0x1b, 0x59, 0x6a, 0x9e, 0x29, 0x0b, 0x67, 0x4a, 0xe0, 0x13, 0xf4, 0x4f, 0x77, - 0x8a, 0x3a, 0x2e, 0x82, 0xfc, 0x5f, 0x59, 0x6a, 0x4e, 0x23, 0x0e, 0xd3, 0x08, 0x75, 0x7f, 0x6d, - 0x22, 0x83, 0xff, 0x84, 0xc0, 0xca, 0x09, 0x11, 0xcf, 0xbf, 0xed, 0x30, 0x09, 0xb4, 0xb1, 0x5a, - 0xc5, 0x41, 0xa3, 0xd8, 0x7b, 0x82, 0x8c, 0x1e, 0x8d, 0x47, 0x09, 0x1b, 0xd0, 0xc5, 0x78, 0x68, - 0x88, 0xf7, 0x44, 0x99, 0x07, 0x15, 0x04, 0xff, 0x17, 0x2d, 0x49, 0x8c, 0x4f, 0xac, 0xe2, 0x21, - 0x6f, 0x58, 0x17, 0xb3, 0xd4, 0xd4, 0x19, 0xa0, 0x93, 0x4c, 0x91, 0xff, 0xf2, 0x00, 0xc4, 0x21, - 0xde, 0x71, 0xfe, 0x6c, 0xe7, 0x8a, 0x1a, 0x03, 0x74, 0x92, 0x3d, 0xc0, 0x39, 0xc0, 0xe7, 0x70, - 0x91, 0x5e, 0xfc, 0x01, 0x9e, 0x83, 0x50, 0x2c, 0xd9, 0xbb, 0x3e, 0x12, 0x67, 0x15, 0xb9, 0x64, - 0x88, 0x77, 0xfd, 0x08, 0x83, 0x7c, 0xc5, 0x1c, 0xe8, 0xaa, 0x85, 0x64, 0xae, 0xa8, 0xc7, 0x2a, - 0x0e, 0x1a, 0x95, 0x8f, 0x99, 0x3b, 0x24, 0xe8, 0xd3, 0xc3, 0x3d, 0x12, 0x1d, 0xe7, 0xa5, 0xbc, - 0x18, 0x33, 0x55, 0x26, 0x54, 0x21, 0x8b, 0xbc, 0x7c, 0xdd, 0x99, 0x79, 0xf5, 0xba, 0x33, 0xf3, - 0xf6, 0x75, 0xa7, 0xf1, 0xf5, 0x69, 0xa7, 0xf1, 0xc3, 0x69, 0xa7, 0xf1, 0xe2, 0xb4, 0xd3, 0x78, - 0x79, 0xda, 0x69, 0xfc, 0x76, 0xda, 0x69, 0xfc, 0x7e, 0xda, 0x99, 0x79, 0x7b, 0xda, 0x69, 0x7c, - 0xf3, 0xa6, 0x33, 0xf3, 0xf2, 0x4d, 0x67, 0xe6, 0xd5, 0x9b, 0xce, 0xcc, 0xe7, 0xbd, 0xbe, 0x47, - 0x0f, 0x93, 0xfd, 0x0d, 0x27, 0x1c, 0xf4, 0xfa, 0x91, 0x7d, 0x60, 0x07, 0x76, 0xcf, 0x0f, 0x9f, - 0x7a, 0xbd, 0xe3, 0xbb, 0xbd, 0xba, 0xdf, 0x68, 0xf7, 0x5b, 0xfc, 0x17, 0xd8, 0xbb, 0x7f, 0x05, - 0x00, 0x00, 0xff, 0xff, 0x6e, 0x53, 0x77, 0xc6, 0xc2, 0x15, 0x00, 0x00, + // 1743 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x58, 0xcb, 0x6f, 0xdb, 0x46, + 0x1a, 0xb7, 0xac, 0x50, 0x56, 0xc6, 0xaf, 0x64, 0xec, 0x24, 0xca, 0x63, 0x45, 0x47, 0x9b, 0x60, + 0xbd, 0xd8, 0x85, 0x85, 0x3c, 0x80, 0xc5, 0x2e, 0x90, 0xc5, 0xae, 0xec, 0x18, 0x08, 0xe0, 0x60, + 0xbd, 0x9f, 0x37, 0x9b, 0xa2, 0x3d, 0xd1, 0xe4, 0x58, 0x66, 0x42, 0x91, 0x32, 0x39, 0x74, 0x62, + 0xb4, 0x40, 0xfb, 0x27, 0xf4, 0x5e, 0xf4, 0x5a, 0xf4, 0x58, 0xa0, 0xa7, 0x9e, 0x7b, 0xc9, 0x31, + 0xc7, 0x9c, 0x88, 0xc6, 0xb9, 0x14, 0x04, 0x0a, 0x04, 0x3d, 0xf7, 0x50, 0xcc, 0x43, 0xe4, 0x0c, + 0x49, 0xc9, 0xca, 0x45, 0xe4, 0xf7, 0xfb, 0x1e, 0x33, 0xfa, 0xe6, 0x7b, 0x0d, 0xd1, 0xda, 0xf0, + 0x79, 0xbf, 0xeb, 0x05, 0xfd, 0x23, 0x6f, 0x10, 0x38, 0xc4, 0xeb, 0x46, 0xd4, 0xa2, 0x91, 0xf8, + 0xdd, 0x18, 0x86, 0x01, 0x0d, 0xb0, 0xc1, 0x89, 0x6b, 0xab, 0xfd, 0xa0, 0x1f, 0x70, 0xa4, 0xcb, + 0xde, 0x04, 0xb3, 0xf3, 0xcd, 0x2c, 0x6a, 0x00, 0x89, 0x62, 0x8f, 0xe2, 0xbf, 0xa3, 0xb9, 0x28, + 0x1e, 0x0c, 0xac, 0xf0, 0xa4, 0x55, 0x5b, 0xab, 0xad, 0xcf, 0xdf, 0x5d, 0xda, 0x10, 0x66, 0xf6, + 0x04, 0xda, 0x5b, 0x7e, 0x95, 0x98, 0x33, 0x69, 0x62, 0x8e, 0xc4, 0x60, 0xf4, 0xc2, 0x54, 0x8f, + 0x62, 0x12, 0xba, 0x24, 0x6c, 0xcd, 0x6a, 0xaa, 0xff, 0x15, 0x68, 0xae, 0x2a, 0xc5, 0x60, 0xf4, + 0x82, 0x1f, 0xa0, 0xa6, 0xeb, 0xf7, 0x49, 0x44, 0x49, 0xd8, 0xaa, 0x73, 0xdd, 0x65, 0xa9, 0xfb, + 0x48, 0xc2, 0xbd, 0x0b, 0x52, 0x39, 0x13, 0x84, 0xec, 0x0d, 0xdf, 0x47, 0x0d, 0xdb, 0xb2, 0x0f, + 0x49, 0xd4, 0x3a, 0xc7, 0x95, 0x17, 0xa5, 0xf2, 0x26, 0x07, 0x7b, 0x8b, 0x52, 0xd5, 0xe0, 0x42, + 0x20, 0x65, 0xf1, 0x1d, 0x64, 0xb8, 0xbe, 0x43, 0x5e, 0xb6, 0x0c, 0xae, 0xb4, 0x90, 0xad, 0xe8, + 0x90, 0x97, 0xb9, 0x0e, 0x17, 0x01, 0xf1, 0xe8, 0x7c, 0x75, 0x0e, 0x35, 0x36, 0x33, 0x6d, 0xfb, + 0x30, 0xf6, 0x9f, 0x4b, 0x37, 0x2d, 0xa8, 0x4b, 0x2a, 0x2b, 0x32, 0x11, 0x10, 0x8f, 0x7c, 0xc1, + 0xd9, 0x49, 0x2a, 0xea, 0x82, 0xec, 0x9f, 0x85, 0xfc, 0x60, 0xa4, 0x5b, 0x74, 0x9d, 0x25, 0xa9, + 0x23, 0x65, 0x40, 0x3e, 0xf1, 0x26, 0x9a, 0xe7, 0x62, 0xe2, 0x4c, 0xa5, 0x53, 0x74, 0xd5, 0x15, + 0xa9, 0xaa, 0x0a, 0x82, 0x4a, 0xe0, 0x6d, 0xb4, 0x70, 0x1c, 0x78, 0xf1, 0x80, 0x48, 0x2b, 0x46, + 0x85, 0x95, 0x55, 0x69, 0x45, 0x93, 0x04, 0x8d, 0x62, 0x76, 0x22, 0x76, 0xca, 0xa3, 0xdd, 0x34, + 0x26, 0xd9, 0x51, 0x25, 0x41, 0xa3, 0xd8, 0x9f, 0xf2, 0xac, 0x7d, 0xe2, 0x49, 0x33, 0x73, 0x93, + 0xfe, 0x94, 0x22, 0x08, 0x2a, 0x81, 0x3f, 0x41, 0x2b, 0xae, 0x1f, 0x51, 0xcb, 0xa7, 0x8f, 0x09, + 0x0d, 0x5d, 0x5b, 0x1a, 0x6b, 0x56, 0x18, 0xbb, 0x2e, 0x8d, 0x55, 0x29, 0x40, 0x15, 0xd8, 0xf9, + 0xa1, 0x81, 0xe6, 0x64, 0x9a, 0xe0, 0x27, 0xe8, 0xca, 0xfe, 0x09, 0x25, 0xd1, 0x6e, 0x18, 0xd8, + 0x24, 0x8a, 0x88, 0xb3, 0x4b, 0xc2, 0x3d, 0x62, 0x07, 0xbe, 0xc3, 0x03, 0xa6, 0xde, 0xbb, 0x9e, + 0x26, 0xe6, 0x38, 0x11, 0x18, 0xc7, 0x60, 0x66, 0x3d, 0xd7, 0xaf, 0x34, 0x3b, 0x9b, 0x9b, 0x1d, + 0x23, 0x02, 0xe3, 0x18, 0xf8, 0x11, 0x5a, 0xa1, 0x01, 0xb5, 0xbc, 0x9e, 0xb6, 0x2c, 0x8f, 0xb9, + 0x7a, 0xef, 0x0a, 0x73, 0x42, 0x05, 0x1b, 0xaa, 0xc0, 0xcc, 0xd4, 0x8e, 0xb6, 0x14, 0x8f, 0x41, + 0xd5, 0x94, 0xce, 0x86, 0x2a, 0x10, 0xaf, 0xa3, 0x26, 0x79, 0x49, 0xec, 0xff, 0xb9, 0x03, 0xc2, + 0xa3, 0xaf, 0xd6, 0x5b, 0x60, 0x05, 0x60, 0x84, 0x41, 0xf6, 0x86, 0xff, 0x82, 0xce, 0x1f, 0xc5, + 0x24, 0x26, 0x5c, 0xb4, 0xc1, 0x45, 0x17, 0xd3, 0xc4, 0xcc, 0x41, 0xc8, 0x5f, 0xf1, 0x06, 0x42, + 0x51, 0xbc, 0x2f, 0x4a, 0x4f, 0xc4, 0xe3, 0xa8, 0xde, 0x5b, 0x4a, 0x13, 0x53, 0x41, 0x41, 0x79, + 0xc7, 0x3b, 0x68, 0x95, 0xef, 0xee, 0xa1, 0x4f, 0x45, 0x38, 0xd2, 0x38, 0xf4, 0x89, 0xc3, 0x83, + 0xa6, 0xde, 0x6b, 0xa5, 0x89, 0x59, 0xc9, 0x87, 0x4a, 0x14, 0x77, 0x50, 0x23, 0x1a, 0x7a, 0x2e, + 0x8d, 0x5a, 0xe7, 0xb9, 0x3e, 0x62, 0xf9, 0x2b, 0x10, 0x90, 0x4f, 0x2e, 0x73, 0x68, 0x85, 0x4e, + 0xd4, 0x42, 0x8a, 0x0c, 0x47, 0x40, 0x3e, 0xb3, 0x5d, 0xed, 0x06, 0x11, 0xdd, 0x76, 0x3d, 0x4a, + 0x42, 0xee, 0xbd, 0xd6, 0x7c, 0x61, 0x57, 0x05, 0x3e, 0x54, 0xa2, 0xf8, 0x73, 0x74, 0x9b, 0xe3, + 0x7b, 0x34, 0x8c, 0x6d, 0x1a, 0x87, 0xc4, 0x79, 0x4c, 0xa8, 0xe5, 0x58, 0xd4, 0x2a, 0x84, 0xc4, + 0x02, 0x37, 0xff, 0xe7, 0x34, 0x31, 0xa7, 0x53, 0x80, 0xe9, 0xc4, 0x3a, 0xdf, 0xcd, 0x22, 0x83, + 0x57, 0x5e, 0x7c, 0x07, 0xcd, 0x73, 0x95, 0x4d, 0x56, 0x33, 0x23, 0x99, 0x2d, 0xcb, 0x2c, 0xab, + 0x15, 0x18, 0x54, 0x02, 0xff, 0x0b, 0x5d, 0x18, 0x66, 0x7f, 0x48, 0xea, 0x89, 0x74, 0x58, 0x4d, + 0x13, 0xb3, 0xc4, 0x83, 0x12, 0x82, 0xff, 0x81, 0x96, 0x84, 0x5f, 0xb7, 0xe2, 0xd0, 0xa2, 0x6e, + 0xe0, 0xcb, 0xd8, 0xc7, 0x69, 0x62, 0x16, 0x38, 0x50, 0xa0, 0xd9, 0xea, 0x71, 0x44, 0x9c, 0x9e, + 0x17, 0x04, 0x03, 0x61, 0x54, 0xf4, 0xa1, 0xa6, 0x58, 0xbd, 0xc8, 0x83, 0x12, 0x82, 0xef, 0xa3, + 0x85, 0x91, 0x97, 0x88, 0x35, 0x88, 0x78, 0xb0, 0xd7, 0x7b, 0x17, 0x58, 0x41, 0x54, 0x71, 0xd0, + 0xa8, 0xce, 0xa7, 0x68, 0x4e, 0x76, 0x56, 0xd6, 0x59, 0x22, 0x1a, 0x84, 0xa4, 0xd0, 0x8c, 0xf6, + 0x18, 0x96, 0x77, 0x16, 0x2e, 0x02, 0xe2, 0x81, 0x1f, 0xa0, 0x65, 0xd9, 0x7d, 0x1f, 0x8e, 0x72, + 0x6c, 0x96, 0x27, 0xce, 0x4a, 0x9a, 0x98, 0x45, 0x16, 0x14, 0x81, 0xce, 0x2f, 0xb3, 0xa8, 0xf9, + 0x28, 0xef, 0xbf, 0x62, 0x67, 0x40, 0x58, 0xe5, 0x14, 0x15, 0xce, 0x50, 0xf6, 0x2f, 0x71, 0xd0, + 0x28, 0xbc, 0x8d, 0xb0, 0x72, 0x88, 0x8f, 0x2d, 0xca, 0x75, 0xc5, 0xb9, 0x5d, 0x4e, 0x13, 0xb3, + 0x82, 0x0b, 0x15, 0x58, 0xb6, 0x7a, 0x8f, 0xd3, 0x91, 0x3c, 0xb9, 0x7c, 0x75, 0x89, 0x83, 0x46, + 0xb1, 0x13, 0xcf, 0x6b, 0xce, 0x1e, 0xf1, 0xa9, 0x2c, 0x51, 0xfc, 0xc4, 0x75, 0x0e, 0x14, 0xe8, + 0xdc, 0xdd, 0xc6, 0xd4, 0xee, 0xfe, 0x27, 0x5a, 0x08, 0x89, 0x7d, 0xfc, 0xd4, 0x72, 0xa9, 0x52, + 0xa4, 0xae, 0xa5, 0x89, 0x79, 0x59, 0xc5, 0xff, 0x1a, 0x0c, 0x5c, 0x4a, 0x06, 0x43, 0x7a, 0x02, + 0x9a, 0x7c, 0xe7, 0x37, 0x03, 0x19, 0xdc, 0x7e, 0xb6, 0x71, 0x19, 0xcb, 0xe4, 0x40, 0xa6, 0x48, + 0xbe, 0xf1, 0x8c, 0x03, 0x05, 0x1a, 0xff, 0x07, 0x5d, 0x52, 0x90, 0xad, 0xe0, 0x85, 0xef, 0x05, + 0x96, 0x93, 0x79, 0xfd, 0x6a, 0x9a, 0x98, 0xd5, 0x02, 0x50, 0x0d, 0xb3, 0x33, 0xb4, 0x35, 0x8c, + 0xff, 0xb9, 0x7a, 0x7e, 0x86, 0x65, 0x2e, 0x54, 0x60, 0xd8, 0x46, 0x57, 0x59, 0x84, 0x9d, 0x00, + 0x39, 0x20, 0x21, 0xf1, 0x6d, 0xe2, 0xe4, 0x15, 0xa3, 0xb5, 0xc8, 0x93, 0xe9, 0x76, 0x9a, 0x98, + 0x37, 0xc7, 0x0a, 0x8d, 0xca, 0x0a, 0x8c, 0xb7, 0x83, 0x37, 0xd1, 0x45, 0xce, 0x7c, 0x12, 0x11, + 0xe7, 0xff, 0x77, 0x1f, 0xfa, 0x7d, 0xd7, 0x27, 0xad, 0x25, 0x6e, 0xfc, 0x52, 0x9a, 0x98, 0x65, + 0x26, 0x94, 0xa1, 0x7c, 0xee, 0x2b, 0x4c, 0x55, 0x0c, 0x1b, 0x33, 0xf7, 0x8d, 0x9c, 0x04, 0xe4, + 0x20, 0xda, 0x26, 0xd4, 0x3e, 0xcc, 0x3a, 0x9a, 0xea, 0x24, 0x8d, 0x0b, 0x15, 0x18, 0xfe, 0x08, + 0xb5, 0xec, 0x80, 0xe7, 0x9c, 0x1b, 0xf8, 0x9b, 0x81, 0x4f, 0xc3, 0xc0, 0xdb, 0xb1, 0x28, 0xf1, + 0xed, 0x13, 0x1e, 0x4f, 0xf5, 0xde, 0x8d, 0x34, 0x31, 0xc7, 0xca, 0xc0, 0x58, 0x0e, 0x76, 0xd0, + 0x8d, 0xa1, 0x3b, 0x24, 0x6c, 0x3c, 0x78, 0x1a, 0x5a, 0xc3, 0x21, 0x09, 0x45, 0x69, 0x22, 0x8e, + 0x68, 0x2a, 0xa2, 0x49, 0xae, 0xa5, 0x89, 0x39, 0x51, 0x0e, 0x26, 0x72, 0xd9, 0x05, 0x81, 0x1d, + 0x51, 0xb0, 0xff, 0x4c, 0x0e, 0x5c, 0xa3, 0x0b, 0xc2, 0x96, 0x40, 0xf3, 0x0b, 0x82, 0x14, 0x83, + 0xd1, 0x4b, 0xe7, 0xd7, 0x26, 0x9a, 0x93, 0x52, 0x7c, 0xb3, 0x21, 0xd9, 0x0d, 0x89, 0xe3, 0xda, + 0x16, 0x25, 0x5b, 0xc4, 0x0e, 0x06, 0xc3, 0x50, 0x74, 0x9b, 0xe0, 0xc5, 0xa8, 0x63, 0x88, 0xcd, + 0x4e, 0x90, 0x83, 0x89, 0x5c, 0xdc, 0x47, 0x7f, 0x18, 0xc7, 0xe7, 0xad, 0x4b, 0xa6, 0xcc, 0xcd, + 0x34, 0x31, 0x27, 0x0b, 0xc2, 0x64, 0x36, 0xfe, 0xba, 0x86, 0xba, 0xe3, 0x24, 0xc6, 0xb4, 0x4d, + 0x99, 0x60, 0xf7, 0xd2, 0xc4, 0xfc, 0x50, 0x55, 0xf8, 0x50, 0x05, 0x96, 0x35, 0xac, 0x5d, 0x66, + 0x3a, 0xdc, 0xc7, 0xa2, 0x56, 0xf2, 0xac, 0x29, 0x31, 0xa1, 0x0c, 0xe1, 0x67, 0xa8, 0xad, 0x81, + 0x65, 0x77, 0x8a, 0x74, 0xe8, 0xa4, 0x89, 0x79, 0x86, 0x24, 0x9c, 0xc1, 0xc7, 0x9f, 0xa1, 0x5b, + 0x9a, 0xc4, 0x38, 0x27, 0x8a, 0x94, 0x59, 0x4f, 0x13, 0x73, 0x2a, 0x79, 0x98, 0x4a, 0x8a, 0x95, + 0xe7, 0x7c, 0xba, 0xe0, 0xbe, 0x9a, 0xcb, 0xcb, 0xb3, 0xce, 0x81, 0x02, 0xcd, 0x3a, 0xd9, 0xd0, + 0xea, 0x93, 0x68, 0xcf, 0xb6, 0xfc, 0x7c, 0xc2, 0xe4, 0x9d, 0x4c, 0xc5, 0x41, 0xa3, 0x58, 0x27, + 0xe7, 0xb4, 0x52, 0xce, 0xc5, 0x68, 0xc9, 0x3b, 0x79, 0x81, 0x05, 0x45, 0x80, 0x0d, 0x92, 0x05, + 0x48, 0xb8, 0x07, 0xe5, 0x83, 0x64, 0x15, 0x1f, 0x2a, 0x51, 0x36, 0xbd, 0x31, 0x7c, 0xd4, 0x8b, + 0xe7, 0xf3, 0xe9, 0x4d, 0x81, 0x41, 0x25, 0xb2, 0x39, 0x80, 0xb9, 0xe0, 0xdf, 0xc7, 0x96, 0xeb, + 0x59, 0xfb, 0x1e, 0x91, 0x83, 0x66, 0x3e, 0x07, 0x68, 0x5c, 0xa8, 0xc0, 0xb2, 0xe6, 0xb6, 0x6b, + 0xf5, 0x89, 0xd6, 0x8e, 0x16, 0x0b, 0xcd, 0xad, 0x28, 0x00, 0xd5, 0x70, 0xe7, 0x7b, 0x03, 0x19, + 0xbc, 0xae, 0xb3, 0x43, 0x3d, 0x24, 0x96, 0x23, 0x8a, 0x3c, 0xf7, 0x8e, 0x32, 0x2c, 0xe8, 0x1c, + 0x28, 0xd0, 0x9a, 0xae, 0xa8, 0xa6, 0x46, 0x85, 0xae, 0xa8, 0x9f, 0x05, 0x9a, 0xe5, 0x9e, 0x53, + 0xca, 0x94, 0x46, 0x9e, 0x7b, 0x25, 0x26, 0x94, 0xa1, 0xa2, 0x11, 0xb5, 0xa2, 0x97, 0x8c, 0x88, + 0x6d, 0x94, 0x21, 0x16, 0x64, 0xc5, 0x7d, 0x34, 0xf3, 0x20, 0x2b, 0xee, 0xa2, 0x08, 0x30, 0x75, + 0xee, 0xe3, 0xad, 0x78, 0xe8, 0xf1, 0xf4, 0x89, 0xd4, 0x18, 0x2d, 0xb0, 0xa0, 0x08, 0xf0, 0x10, + 0x2f, 0xdc, 0x73, 0x90, 0x12, 0xe2, 0x85, 0x2b, 0x4e, 0x11, 0xc0, 0x43, 0xb4, 0x96, 0x39, 0x76, + 0x5c, 0x35, 0x10, 0x91, 0x7a, 0x2b, 0x4d, 0xcc, 0x33, 0x65, 0xe1, 0x4c, 0x09, 0x7c, 0x82, 0xfe, + 0xe8, 0x4c, 0x51, 0xc7, 0x45, 0x90, 0xff, 0x29, 0x4d, 0xcc, 0x69, 0xc4, 0x61, 0x1a, 0xa1, 0xce, + 0x8f, 0x75, 0x64, 0xf0, 0x2f, 0x18, 0xac, 0x9c, 0x10, 0x71, 0xfb, 0xdc, 0x0e, 0x62, 0x5f, 0x1b, + 0xcb, 0x55, 0x1c, 0x34, 0x8a, 0x5d, 0x67, 0xc8, 0xe8, 0xce, 0x7a, 0x14, 0xb3, 0x01, 0x5f, 0x8c, + 0x87, 0x86, 0xb8, 0xce, 0x14, 0x79, 0x50, 0x42, 0xf0, 0xdf, 0xd0, 0xa2, 0xc4, 0xf8, 0xc4, 0x2a, + 0xbe, 0x23, 0x18, 0xbd, 0x8b, 0x69, 0x62, 0xea, 0x0c, 0xd0, 0x49, 0xa6, 0xc8, 0x3f, 0x7c, 0x00, + 0xb1, 0x89, 0x7b, 0x9c, 0x7d, 0x35, 0xe0, 0x8a, 0x1a, 0x03, 0x74, 0x92, 0xdd, 0xff, 0x39, 0xc0, + 0xe7, 0x78, 0x91, 0x5e, 0xfc, 0xfe, 0x9f, 0x81, 0x90, 0xbf, 0xe2, 0x75, 0xd4, 0x0c, 0xc5, 0x5e, + 0x45, 0x2e, 0x19, 0xe2, 0xb3, 0xc2, 0x08, 0x83, 0xec, 0x8d, 0x39, 0xd0, 0x51, 0x0b, 0xc9, 0x5c, + 0x5e, 0x8f, 0x55, 0x1c, 0x34, 0x2a, 0x1b, 0x33, 0x77, 0x88, 0xdf, 0xa7, 0x87, 0x7b, 0x24, 0x3c, + 0xce, 0x4a, 0x79, 0x3e, 0x66, 0xaa, 0x4c, 0x28, 0x43, 0x3d, 0xf2, 0xfa, 0x6d, 0x7b, 0xe6, 0xcd, + 0xdb, 0xf6, 0xcc, 0xfb, 0xb7, 0xed, 0xda, 0x17, 0xa7, 0xed, 0xda, 0xb7, 0xa7, 0xed, 0xda, 0xab, + 0xd3, 0x76, 0xed, 0xf5, 0x69, 0xbb, 0xf6, 0xd3, 0x69, 0xbb, 0xf6, 0xf3, 0x69, 0x7b, 0xe6, 0xfd, + 0x69, 0xbb, 0xf6, 0xe5, 0xbb, 0xf6, 0xcc, 0xeb, 0x77, 0xed, 0x99, 0x37, 0xef, 0xda, 0x33, 0x1f, + 0x77, 0xfb, 0x2e, 0x3d, 0x8c, 0xf7, 0x37, 0xec, 0x60, 0xd0, 0xed, 0x87, 0xd6, 0x81, 0xe5, 0x5b, + 0x5d, 0x2f, 0x78, 0xee, 0x76, 0x8f, 0xef, 0x75, 0xab, 0x3e, 0x11, 0xef, 0x37, 0xf8, 0x07, 0xe0, + 0x7b, 0xbf, 0x07, 0x00, 0x00, 0xff, 0xff, 0x52, 0x6a, 0xf1, 0xfe, 0x41, 0x16, 0x00, 0x00, } func (this *Result) Equal(that interface{}) bool { @@ -1366,6 +1391,9 @@ func (this *Querier) Equal(that interface{}) bool { if !this.Store.Equal(&that1.Store) { return false } + if this.QuerierExecTime != that1.QuerierExecTime { + return false + } return true } func (this *Ingester) Equal(that interface{}) bool { @@ -1402,6 +1430,9 @@ func (this *Ingester) Equal(that interface{}) bool { if !this.Store.Equal(&that1.Store) { return false } + if this.RecvWaitTime != that1.RecvWaitTime { + return false + } return true } func (this *Store) Equal(that interface{}) bool { @@ -1678,9 +1709,10 @@ func (this *Querier) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 6) s = append(s, "&stats.Querier{") s = append(s, "Store: "+strings.Replace(this.Store.GoString(), `&`, ``, 1)+",\n") + s = append(s, "QuerierExecTime: "+fmt.Sprintf("%#v", this.QuerierExecTime)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1688,13 +1720,14 @@ func (this *Ingester) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 9) + s := make([]string, 0, 10) s = append(s, "&stats.Ingester{") s = append(s, "TotalReached: "+fmt.Sprintf("%#v", this.TotalReached)+",\n") s = append(s, "TotalChunksMatched: "+fmt.Sprintf("%#v", this.TotalChunksMatched)+",\n") s = append(s, "TotalBatches: "+fmt.Sprintf("%#v", this.TotalBatches)+",\n") s = append(s, "TotalLinesSent: "+fmt.Sprintf("%#v", this.TotalLinesSent)+",\n") s = append(s, "Store: "+strings.Replace(this.Store.GoString(), `&`, ``, 1)+",\n") + s = append(s, "RecvWaitTime: "+fmt.Sprintf("%#v", this.RecvWaitTime)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2116,6 +2149,12 @@ func (m *Querier) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.QuerierExecTime != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.QuerierExecTime)))) + i-- + dAtA[i] = 0x11 + } { size, err := m.Store.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -2149,6 +2188,12 @@ func (m *Ingester) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.RecvWaitTime != 0 { + i -= 8 + encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.RecvWaitTime)))) + i-- + dAtA[i] = 0x31 + } { size, err := m.Store.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -2626,6 +2671,9 @@ func (m *Querier) Size() (n int) { _ = l l = m.Store.Size() n += 1 + l + sovStats(uint64(l)) + if m.QuerierExecTime != 0 { + n += 9 + } return n } @@ -2649,6 +2697,9 @@ func (m *Ingester) Size() (n int) { } l = m.Store.Size() n += 1 + l + sovStats(uint64(l)) + if m.RecvWaitTime != 0 { + n += 9 + } return n } @@ -2884,6 +2935,7 @@ func (this *Querier) String() string { } s := strings.Join([]string{`&Querier{`, `Store:` + strings.Replace(strings.Replace(this.Store.String(), "Store", "Store", 1), `&`, ``, 1) + `,`, + `QuerierExecTime:` + fmt.Sprintf("%v", this.QuerierExecTime) + `,`, `}`, }, "") return s @@ -2898,6 +2950,7 @@ func (this *Ingester) String() string { `TotalBatches:` + fmt.Sprintf("%v", this.TotalBatches) + `,`, `TotalLinesSent:` + fmt.Sprintf("%v", this.TotalLinesSent) + `,`, `Store:` + strings.Replace(strings.Replace(this.Store.String(), "Store", "Store", 1), `&`, ``, 1) + `,`, + `RecvWaitTime:` + fmt.Sprintf("%v", this.RecvWaitTime) + `,`, `}`, }, "") return s @@ -3997,6 +4050,17 @@ func (m *Querier) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field QuerierExecTime", wireType) + } + var v uint64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + m.QuerierExecTime = float64(math.Float64frombits(v)) default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) @@ -4159,6 +4223,17 @@ func (m *Ingester) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field RecvWaitTime", wireType) + } + var v uint64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + m.RecvWaitTime = float64(math.Float64frombits(v)) default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) diff --git a/pkg/logqlmodel/stats/stats.proto b/pkg/logqlmodel/stats/stats.proto index 00bb818e45..b148713945 100644 --- a/pkg/logqlmodel/stats/stats.proto +++ b/pkg/logqlmodel/stats/stats.proto @@ -121,6 +121,10 @@ message Querier { (gogoproto.nullable) = false, (gogoproto.jsontag) = "store" ]; + // Querier execution time in seconds. + // Total time the querier spent executing the query. This is separate from execTime which tracks overall execution time. + // Grafana expects time values to be returned in seconds as float. + double querierExecTime = 2 [(gogoproto.jsontag) = "querierExecTime"]; } message Ingester { @@ -137,6 +141,11 @@ message Ingester { (gogoproto.nullable) = false, (gogoproto.jsontag) = "store" ]; + + // Total time querier spent waiting on ingester gRPC Recv(), in seconds. + // Omitted from JSON when zero for backwards compatibility. + // Grafana expects time values to be returned in seconds as float + double recvWaitTime = 6 [(gogoproto.jsontag) = "recvWaitTime,omitempty"]; } message Store { diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index 47127c4932..039bda4486 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -1955,6 +1955,7 @@ var ( "totalReached": 10 }, "querier": { + "querierExecTime": 0, "store" : { "chunk": { "compressedBytes": 11, diff --git a/pkg/querier/queryrange/prometheus_test.go b/pkg/querier/queryrange/prometheus_test.go index aba98bea99..1a73ba7517 100644 --- a/pkg/querier/queryrange/prometheus_test.go +++ b/pkg/querier/queryrange/prometheus_test.go @@ -63,6 +63,7 @@ var emptyStats = `"stats": { "totalReached": 0 }, "querier": { + "querierExecTime": 0, "store": { "chunksDownloadTime": 0, "congestionControlLatency": 0, diff --git a/pkg/util/marshal/legacy/marshal_test.go b/pkg/util/marshal/legacy/marshal_test.go index 67c68842f8..a3c3f6b8a0 100644 --- a/pkg/util/marshal/legacy/marshal_test.go +++ b/pkg/util/marshal/legacy/marshal_test.go @@ -106,6 +106,7 @@ var queryTests = []struct { "totalReached": 0 }, "querier": { + "querierExecTime": 0, "store": { "chunksDownloadTime": 0, "congestionControlLatency": 0, diff --git a/pkg/util/marshal/marshal_test.go b/pkg/util/marshal/marshal_test.go index 75d392a0b4..0e1a87fc2c 100644 --- a/pkg/util/marshal/marshal_test.go +++ b/pkg/util/marshal/marshal_test.go @@ -77,6 +77,7 @@ const emptyStats = `{ "totalReached": 0 }, "querier": { + "querierExecTime": 0, "store": { "chunksDownloadTime": 0, "congestionControlLatency": 0,