From 5b97fcfd93ae0aa341284b6557697495c445c196 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 22 Nov 2023 08:29:02 +0100 Subject: [PATCH] Send query plan to querier. (#11246) **What this PR does / why we need it**: Following https://github.com/grafana/loki/pull/11123 and in order to enable https://github.com/grafana/loki/pull/10417 the query frontend should send the serialized LogQL AST instead of the query string to the queriers. This enables the frontend to change the AST and inject expressions that are not expressible in LogQL. **Checklist** - [ ] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - [ ] Documentation added - [x] Tests updated - [ ] `CHANGELOG.md` updated - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213) - [ ] If the change is deprecating or removing a configuration option, update the `deprecated-config.yaml` and `deleted-config.yaml` files respectively in the `tools/deprecated-config-checker` directory. [Example PR](https://github.com/grafana/loki/pull/10840/commits/0d4416a4b03739583349934b96f272fb4f685d15) --------- Signed-off-by: Callum Styan Co-authored-by: Callum Styan --- pkg/logcli/client/file.go | 10 +- pkg/logcli/query/query.go | 22 +- pkg/logcli/query/query_test.go | 5 +- pkg/logproto/indexgateway.pb.go | 50 ++- pkg/logproto/indexgateway.proto | 1 - pkg/logproto/sketch.pb.go | 78 ++-- pkg/logproto/sketch.proto | 1 - pkg/logql/blocker.go | 4 +- pkg/logql/blocker_test.go | 13 +- pkg/logql/downstream.go | 46 ++- pkg/logql/downstream_test.go | 25 +- pkg/logql/engine.go | 27 +- pkg/logql/engine_test.go | 160 +++----- pkg/logql/evaluator.go | 69 +++- pkg/logql/evaluator_test.go | 4 +- pkg/logql/explain_test.go | 10 +- pkg/logql/metrics.go | 12 +- pkg/logql/metrics_test.go | 49 ++- pkg/logql/rangemapper.go | 8 +- pkg/logql/rangemapper_test.go | 24 +- pkg/logql/shardmapper.go | 7 +- pkg/logql/shardmapper_test.go | 12 +- pkg/logql/syntax/parser.go | 8 + pkg/logql/test_utils.go | 12 +- pkg/querier/plan/plan.go | 101 +++++ pkg/querier/plan/plan_test.go | 26 ++ pkg/querier/queryrange/codec.go | 66 +++- pkg/querier/queryrange/codec_test.go | 19 +- pkg/querier/queryrange/downstreamer.go | 21 +- pkg/querier/queryrange/downstreamer_test.go | 58 ++- pkg/querier/queryrange/marshal.go | 24 ++ pkg/querier/queryrange/queryrange.pb.go | 345 ++++++++++++------ pkg/querier/queryrange/queryrange.proto | 2 + pkg/querier/queryrange/querysharding.go | 14 +- pkg/querier/queryrange/querysharding_test.go | 40 +- pkg/querier/queryrange/roundtrip_test.go | 20 +- pkg/querier/queryrange/split_by_interval.go | 2 + pkg/querier/queryrange/split_by_range.go | 16 +- pkg/querier/queryrange/split_by_range_test.go | 26 +- pkg/querier/queryrange/stats.go | 6 +- pkg/querier/queryrange/stats_test.go | 12 +- pkg/querier/worker/util_test.go | 2 +- pkg/ruler/evaluator_local.go | 5 +- 43 files changed, 932 insertions(+), 530 deletions(-) create mode 100644 pkg/querier/plan/plan.go create mode 100644 pkg/querier/plan/plan_test.go diff --git a/pkg/logcli/client/file.go b/pkg/logcli/client/file.go index 45681c36c2..82274ef79f 100644 --- a/pkg/logcli/client/file.go +++ b/pkg/logcli/client/file.go @@ -69,7 +69,7 @@ func (f *FileClient) Query(q string, limit int, t time.Time, direction logproto. ctx = user.InjectOrgID(ctx, f.orgID) - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q, t, t, 0, @@ -78,6 +78,9 @@ func (f *FileClient) Query(q string, limit int, t time.Time, direction logproto. uint32(limit), nil, ) + if err != nil { + return nil, fmt.Errorf("failed to parse query: %w", err) + } query := f.engine.Query(params) @@ -106,7 +109,7 @@ func (f *FileClient) QueryRange(queryStr string, limit int, start, end time.Time ctx = user.InjectOrgID(ctx, f.orgID) - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( queryStr, start, end, @@ -116,6 +119,9 @@ func (f *FileClient) QueryRange(queryStr string, limit int, start, end time.Time uint32(limit), nil, ) + if err != nil { + return nil, err + } query := f.engine.Query(params) diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index 6a71f0979a..fc5be5f393 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -451,7 +451,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string var query logql.Query if q.isInstant() { - query = eng.Query(logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q.QueryString, q.Start, q.Start, @@ -460,9 +460,14 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string q.resultsDirection(), uint32(q.Limit), nil, - )) + ) + if err != nil { + return err + } + + query = eng.Query(params) } else { - query = eng.Query(logql.NewLiteralParams( + params, err := logql.NewLiteralParams( q.QueryString, q.Start, q.End, @@ -471,7 +476,16 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string q.resultsDirection(), uint32(q.Limit), nil, - )) + ) + if err != nil { + return err + } + + query = eng.Query(params) + } + + if err != nil { + return err } // execute the query diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index 72886fb846..1b4c18f526 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -425,7 +425,10 @@ func (t *testQueryClient) Query(_ string, _ int, _ time.Time, _ logproto.Directi func (t *testQueryClient) QueryRange(queryStr string, limit int, from, through time.Time, direction logproto.Direction, step, interval time.Duration, _ bool) (*loghttp.QueryResponse, error) { ctx := user.InjectOrgID(context.Background(), "fake") - params := logql.NewLiteralParams(queryStr, from, through, step, interval, direction, uint32(limit), nil) + params, err := logql.NewLiteralParams(queryStr, from, through, step, interval, direction, uint32(limit), nil) + if err != nil { + return nil, err + } v, err := t.engine.Query(params).Exec(ctx) if err != nil { diff --git a/pkg/logproto/indexgateway.pb.go b/pkg/logproto/indexgateway.pb.go index e8b569ea07..86b2665e86 100644 --- a/pkg/logproto/indexgateway.pb.go +++ b/pkg/logproto/indexgateway.pb.go @@ -6,7 +6,6 @@ package logproto import ( context "context" fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -28,31 +27,30 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("pkg/logproto/indexgateway.proto", fileDescriptor_d27585148d0a52c8) } var fileDescriptor_d27585148d0a52c8 = []byte{ - // 372 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xc1, 0x4a, 0xfb, 0x30, - 0x1c, 0xc7, 0x1b, 0xf8, 0xf3, 0x47, 0xa3, 0x78, 0x08, 0xc2, 0x46, 0xa7, 0x11, 0xc4, 0x83, 0x5e, - 0x56, 0xd1, 0x17, 0x10, 0x85, 0x95, 0xc1, 0x14, 0x9c, 0xb0, 0xc3, 0x0e, 0x62, 0x3a, 0x7f, 0xeb, - 0xca, 0xba, 0xa6, 0xb6, 0x29, 0xba, 0x9b, 0x8f, 0xe0, 0x63, 0xf8, 0x10, 0x3e, 0x80, 0xc7, 0x1d, - 0x77, 0x74, 0xdd, 0xc5, 0xe3, 0x1e, 0x41, 0x9a, 0xd0, 0x2d, 0x9b, 0x1d, 0x78, 0x6a, 0xfa, 0xf9, - 0x7e, 0xf3, 0xf9, 0xd1, 0xa4, 0xf8, 0x20, 0xec, 0xbb, 0x96, 0xcf, 0xdd, 0x30, 0xe2, 0x82, 0x5b, - 0x5e, 0xf0, 0x08, 0x2f, 0x2e, 0x13, 0xf0, 0xcc, 0x86, 0x55, 0x89, 0xc8, 0x8e, 0xce, 0x42, 0xc7, - 0xdc, 0x75, 0xb9, 0xcb, 0x55, 0x3b, 0x5b, 0xa9, 0x96, 0x59, 0x59, 0xd2, 0xe4, 0x0b, 0x15, 0x9e, - 0x7d, 0xfc, 0xc3, 0xdb, 0xf5, 0xcc, 0x62, 0x2b, 0x0b, 0xa9, 0x63, 0x7c, 0x9b, 0x40, 0x34, 0x94, - 0x90, 0x54, 0xaa, 0xf3, 0xfe, 0x82, 0x36, 0xe1, 0x29, 0x81, 0x58, 0x98, 0x7b, 0xc5, 0x61, 0x1c, - 0xf2, 0x20, 0x86, 0x53, 0x44, 0x1a, 0x78, 0xcb, 0x06, 0x71, 0xd5, 0x4b, 0x82, 0x7e, 0x13, 0xba, - 0x44, 0xab, 0x6b, 0x38, 0x97, 0xed, 0xaf, 0x49, 0x95, 0xed, 0xd0, 0x20, 0x35, 0xbc, 0x69, 0x83, - 0xb8, 0x83, 0xc8, 0x83, 0x98, 0x98, 0x4b, 0x6d, 0x05, 0x73, 0x53, 0xa5, 0x30, 0x9b, 0x7b, 0xee, - 0x71, 0xa9, 0xc1, 0x1c, 0xf0, 0x6f, 0xd8, 0x00, 0xe2, 0x1a, 0x8f, 0xae, 0x41, 0x44, 0x5e, 0x27, - 0x7b, 0x23, 0xc7, 0x8b, 0x9d, 0x6b, 0x2a, 0xf9, 0x8c, 0xd2, 0x4a, 0x53, 0xf3, 0x3f, 0xe0, 0xb2, - 0x44, 0x2d, 0xe6, 0x27, 0xab, 0x03, 0x4e, 0x56, 0xb6, 0x15, 0x74, 0xfe, 0x30, 0xc1, 0xc6, 0x1b, - 0xd9, 0x87, 0x09, 0x26, 0x62, 0xfd, 0x82, 0xe4, 0xf1, 0x4b, 0x5a, 0x70, 0x41, 0x7a, 0x38, 0x17, - 0x5d, 0xc8, 0x23, 0x6d, 0x71, 0x3f, 0x19, 0x00, 0xd1, 0x06, 0x2a, 0x92, 0x5b, 0xca, 0xbf, 0x83, - 0xdc, 0x70, 0xd9, 0x1e, 0x4d, 0xa8, 0x31, 0x9e, 0x50, 0x63, 0x36, 0xa1, 0xe8, 0x35, 0xa5, 0xe8, - 0x3d, 0xa5, 0xe8, 0x33, 0xa5, 0x68, 0x94, 0x52, 0xf4, 0x95, 0x52, 0xf4, 0x9d, 0x52, 0x63, 0x96, - 0x52, 0xf4, 0x36, 0xa5, 0xc6, 0x68, 0x4a, 0x8d, 0xf1, 0x94, 0x1a, 0xed, 0x23, 0xd7, 0x13, 0xbd, - 0xc4, 0xa9, 0x76, 0xf8, 0xc0, 0x72, 0x23, 0xd6, 0x65, 0x01, 0xb3, 0x7c, 0xde, 0xf7, 0x2c, 0xfd, - 0x4f, 0x75, 0xfe, 0xcb, 0xc7, 0xf9, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x79, 0xe4, 0x24, 0x34, - 0x07, 0x03, 0x00, 0x00, + // 361 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xb1, 0x4e, 0xc2, 0x40, + 0x18, 0xc7, 0xef, 0x12, 0x63, 0xf4, 0x34, 0x0e, 0xb7, 0x40, 0x40, 0xcf, 0xc4, 0x38, 0xe8, 0x42, + 0x8d, 0xbe, 0x80, 0xd1, 0x84, 0x86, 0x04, 0x4d, 0xc4, 0x84, 0x81, 0xc1, 0x78, 0xc5, 0x8f, 0xd2, + 0x50, 0x7a, 0xb5, 0xbd, 0x46, 0xd9, 0x7c, 0x04, 0x1f, 0xc3, 0x87, 0xf0, 0x01, 0x1c, 0x19, 0x19, + 0xe5, 0x58, 0x1c, 0x79, 0x04, 0xc3, 0x35, 0x85, 0x03, 0x4b, 0xe2, 0x04, 0xfd, 0xfd, 0x7f, 0xdf, + 0xff, 0x4b, 0xef, 0x4a, 0x0e, 0xc3, 0x9e, 0x6b, 0xf9, 0xc2, 0x0d, 0x23, 0x21, 0x85, 0xe5, 0x05, + 0x4f, 0xf0, 0xea, 0x72, 0x09, 0x2f, 0x7c, 0x50, 0xd1, 0x88, 0xee, 0x99, 0x2c, 0x74, 0x4a, 0xe5, + 0xa5, 0x81, 0xec, 0x4f, 0x2a, 0x9f, 0x7f, 0x6e, 0x90, 0xdd, 0xda, 0xcc, 0xb7, 0x53, 0x9f, 0xd6, + 0x08, 0xb9, 0x4b, 0x20, 0x1a, 0x68, 0x48, 0xcb, 0x95, 0xb9, 0xbf, 0xa0, 0x0d, 0x78, 0x4e, 0x20, + 0x96, 0xa5, 0xfd, 0xfc, 0x30, 0x0e, 0x45, 0x10, 0xc3, 0x19, 0xa6, 0x75, 0xb2, 0x63, 0x83, 0xbc, + 0xee, 0x26, 0x41, 0xaf, 0x01, 0x1d, 0x6a, 0xe8, 0x06, 0xce, 0xca, 0x0e, 0xd6, 0xa4, 0x69, 0xdb, + 0x11, 0xa2, 0x55, 0xb2, 0x6d, 0x83, 0xbc, 0x87, 0xc8, 0x83, 0x98, 0x96, 0x96, 0xec, 0x14, 0x66, + 0x4d, 0xe5, 0xdc, 0x6c, 0xde, 0xf3, 0x40, 0x0a, 0x75, 0xee, 0x80, 0x7f, 0xcb, 0xfb, 0x10, 0x57, + 0x45, 0x74, 0x03, 0x32, 0xf2, 0xda, 0xb3, 0x27, 0x7a, 0xb2, 0x98, 0x5c, 0xa3, 0x64, 0x3b, 0x0a, + 0x2b, 0xa6, 0xd1, 0xff, 0x48, 0x8a, 0x1a, 0x35, 0xb9, 0x9f, 0xac, 0x2e, 0x38, 0x5d, 0x19, 0xcb, + 0x71, 0xfe, 0xb1, 0xc1, 0x26, 0x5b, 0xb3, 0x17, 0x93, 0x5c, 0xc6, 0xe6, 0x05, 0xe9, 0xe3, 0xd7, + 0x34, 0xe7, 0x82, 0xcc, 0x70, 0x5e, 0x74, 0xa9, 0x8f, 0xb4, 0x29, 0xfc, 0xa4, 0x0f, 0xd4, 0x58, + 0x98, 0x92, 0xac, 0xa5, 0xf8, 0x37, 0xc8, 0x1a, 0xae, 0x5a, 0xc3, 0x31, 0x43, 0xa3, 0x31, 0x43, + 0xd3, 0x31, 0xc3, 0x6f, 0x8a, 0xe1, 0x0f, 0xc5, 0xf0, 0x97, 0x62, 0x78, 0xa8, 0x18, 0xfe, 0x56, + 0x0c, 0xff, 0x28, 0x86, 0xa6, 0x8a, 0xe1, 0xf7, 0x09, 0x43, 0xc3, 0x09, 0x43, 0xa3, 0x09, 0x43, + 0xad, 0x63, 0xd7, 0x93, 0xdd, 0xc4, 0xa9, 0xb4, 0x45, 0xdf, 0x72, 0x23, 0xde, 0xe1, 0x01, 0xb7, + 0x7c, 0xd1, 0xf3, 0x2c, 0xf3, 0x4b, 0x75, 0x36, 0xf5, 0xcf, 0xc5, 0x6f, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x7a, 0x1a, 0x28, 0xb4, 0xf1, 0x02, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/pkg/logproto/indexgateway.proto b/pkg/logproto/indexgateway.proto index 9271ee9b2b..af34e03a27 100644 --- a/pkg/logproto/indexgateway.proto +++ b/pkg/logproto/indexgateway.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package indexgatewaypb; -import "gogoproto/gogo.proto"; import "pkg/logproto/logproto.proto"; option go_package = "github.com/grafana/loki/pkg/logproto"; diff --git a/pkg/logproto/sketch.pb.go b/pkg/logproto/sketch.pb.go index 4a56552d98..c555d64d55 100644 --- a/pkg/logproto/sketch.pb.go +++ b/pkg/logproto/sketch.pb.go @@ -7,7 +7,6 @@ import ( bytes "bytes" encoding_binary "encoding/binary" fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -657,47 +656,46 @@ func init() { func init() { proto.RegisterFile("pkg/logproto/sketch.proto", fileDescriptor_7f9fd40e59b87ff3) } var fileDescriptor_7f9fd40e59b87ff3 = []byte{ - // 632 bytes of a gzipped FileDescriptorProto + // 623 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x41, 0x4f, 0xd4, 0x4e, - 0x14, 0xef, 0xfc, 0x77, 0xff, 0xcb, 0xf2, 0x16, 0x88, 0x8e, 0xc4, 0xac, 0xc5, 0x4c, 0xd6, 0xc6, + 0x14, 0xef, 0xfc, 0x77, 0xff, 0xcb, 0xf2, 0x16, 0x88, 0x8e, 0xc4, 0xd4, 0xc5, 0x4c, 0xd6, 0xc6, 0x28, 0xd1, 0xb8, 0x9b, 0x40, 0x42, 0x38, 0x83, 0x07, 0x12, 0x45, 0x71, 0x20, 0xc6, 0x70, 0x31, - 0xa5, 0x1d, 0xba, 0x93, 0x6d, 0x3b, 0x4d, 0x67, 0x16, 0xf0, 0xe6, 0x27, 0x30, 0xc6, 0x4f, 0xe1, - 0xd5, 0x8f, 0xe0, 0xcd, 0x23, 0x47, 0x8e, 0x52, 0x2e, 0x1e, 0xf9, 0x08, 0x66, 0x66, 0xda, 0x85, - 0x2e, 0x31, 0x7a, 0xda, 0xf7, 0x7e, 0xef, 0xf7, 0x7e, 0xf3, 0x9b, 0x79, 0x7d, 0x0b, 0xf7, 0xb2, - 0x51, 0x34, 0x88, 0x45, 0x94, 0xe5, 0x42, 0x89, 0x81, 0x1c, 0x31, 0x15, 0x0c, 0xfb, 0x26, 0xc1, - 0xed, 0x0a, 0x76, 0x17, 0x23, 0x11, 0x09, 0xcb, 0xd0, 0x91, 0xad, 0xbb, 0x4b, 0xb5, 0xd6, 0x2a, - 0xb0, 0x45, 0xef, 0x15, 0x2c, 0xbe, 0x19, 0xfb, 0xa9, 0xe2, 0x31, 0xdb, 0x35, 0xa2, 0xdb, 0xbe, - 0xca, 0xf9, 0x09, 0x5e, 0x83, 0xd6, 0x91, 0x1f, 0x8f, 0x99, 0xec, 0xa2, 0x5e, 0x63, 0xb9, 0xb3, - 0x42, 0xfa, 0x93, 0xc6, 0x3a, 0xff, 0x2d, 0x0b, 0x94, 0xc8, 0x69, 0xc9, 0xf6, 0x76, 0xa6, 0xf5, - 0x6c, 0x1d, 0xaf, 0xc3, 0x8c, 0xf4, 0x93, 0x2c, 0xfe, 0xbb, 0xe0, 0xae, 0xa1, 0xd1, 0x8a, 0xee, - 0x7d, 0x42, 0xd3, 0x92, 0x96, 0x81, 0x1f, 0x01, 0x3a, 0xec, 0xa2, 0x1e, 0x5a, 0xee, 0xac, 0x74, - 0xff, 0x24, 0x46, 0xd1, 0x21, 0x7e, 0x00, 0x73, 0x8a, 0x27, 0x4c, 0x2a, 0x3f, 0xc9, 0xde, 0x27, - 0xb2, 0xfb, 0x5f, 0x0f, 0x2d, 0x37, 0x68, 0x67, 0x82, 0x6d, 0x4b, 0xfc, 0x14, 0x5a, 0x09, 0x53, - 0x39, 0x0f, 0xba, 0x0d, 0x63, 0xee, 0xce, 0x95, 0xde, 0x4b, 0xff, 0x80, 0xc5, 0x3b, 0x3e, 0xcf, - 0x69, 0x49, 0xf1, 0x22, 0x58, 0xa8, 0x1f, 0x82, 0x9f, 0xc1, 0x8c, 0x0a, 0x79, 0xc4, 0xa4, 0x2a, - 0xfd, 0xdc, 0xbe, 0xea, 0xdf, 0x7b, 0x6e, 0x0a, 0x5b, 0x0e, 0xad, 0x38, 0xf8, 0x3e, 0xb4, 0xc3, - 0xd0, 0x8e, 0xd0, 0x98, 0x99, 0xdb, 0x72, 0xe8, 0x04, 0xd9, 0x68, 0x43, 0xcb, 0x46, 0xde, 0x77, - 0x04, 0x33, 0x65, 0x3b, 0xbe, 0x05, 0x8d, 0x84, 0xa7, 0x46, 0x1e, 0x51, 0x1d, 0x1a, 0xc4, 0x3f, - 0x31, 0x02, 0x1a, 0xf1, 0x4f, 0x70, 0x0f, 0x3a, 0x81, 0x48, 0xb2, 0x9c, 0x49, 0xc9, 0x45, 0xda, - 0x6d, 0x98, 0xca, 0x75, 0x08, 0xaf, 0xc3, 0x6c, 0x96, 0x8b, 0x80, 0x49, 0xc9, 0xc2, 0x6e, 0xd3, - 0x5c, 0xd5, 0xbd, 0x61, 0xb5, 0xbf, 0xc9, 0x52, 0x95, 0x0b, 0x1e, 0xd2, 0x2b, 0xb2, 0xbb, 0x06, - 0xed, 0x0a, 0xc6, 0x18, 0x9a, 0x09, 0xf3, 0x2b, 0x33, 0x26, 0xc6, 0x77, 0xa1, 0x75, 0xcc, 0x78, - 0x34, 0x54, 0xa5, 0xa1, 0x32, 0xf3, 0xde, 0xc1, 0xc2, 0xa6, 0x18, 0xa7, 0x6a, 0x9b, 0xa7, 0xe5, - 0x63, 0x2d, 0xc2, 0xff, 0x21, 0xcb, 0xd4, 0xd0, 0xb4, 0xcf, 0x53, 0x9b, 0x68, 0xf4, 0x98, 0x87, - 0xca, 0x3e, 0xc8, 0x3c, 0xb5, 0x09, 0x76, 0xa1, 0x1d, 0xe8, 0x6e, 0x96, 0x4b, 0x33, 0x99, 0x79, - 0x3a, 0xc9, 0xbd, 0x6f, 0x08, 0x9a, 0x7b, 0x22, 0x7b, 0x81, 0x9f, 0x40, 0x23, 0x48, 0xe4, 0xcd, - 0x2f, 0xa1, 0x7e, 0x2e, 0xd5, 0x24, 0xfc, 0x18, 0x9a, 0x31, 0x97, 0xda, 0xe4, 0xd4, 0x98, 0xb5, - 0x52, 0xdf, 0x8c, 0xd9, 0x10, 0xf4, 0x5b, 0x0e, 0x3f, 0x64, 0x2c, 0x8f, 0x45, 0x14, 0x8b, 0xc8, - 0xbc, 0xe5, 0x1c, 0xbd, 0x0e, 0xb9, 0x2b, 0xd0, 0xd4, 0x7c, 0xed, 0x9c, 0x1d, 0xb1, 0xd4, 0x8e, - 0x7e, 0x96, 0xda, 0x44, 0xa3, 0xc6, 0x69, 0x75, 0x1f, 0x93, 0x78, 0x5f, 0x10, 0x80, 0x3e, 0xa9, - 0x5c, 0xb2, 0xd5, 0xa9, 0x25, 0x5b, 0xaa, 0xfb, 0xb1, 0xac, 0x7e, 0x7d, 0xc3, 0xdc, 0xd7, 0xd0, - 0x2a, 0x77, 0xca, 0x83, 0xa6, 0x12, 0xd9, 0xa8, 0xbc, 0xf9, 0x42, 0xbd, 0x99, 0x9a, 0xda, 0x3f, - 0x7c, 0xfc, 0x1b, 0xfb, 0xa7, 0xe7, 0xc4, 0x39, 0x3b, 0x27, 0xce, 0xe5, 0x39, 0x41, 0x1f, 0x0b, - 0x82, 0xbe, 0x16, 0x04, 0xfd, 0x28, 0x08, 0x3a, 0x2d, 0x08, 0xfa, 0x59, 0x10, 0xf4, 0xab, 0x20, - 0xce, 0x65, 0x41, 0xd0, 0xe7, 0x0b, 0xe2, 0x9c, 0x5e, 0x10, 0xe7, 0xec, 0x82, 0x38, 0xfb, 0x0f, - 0x23, 0xae, 0x86, 0xe3, 0x83, 0x7e, 0x20, 0x92, 0x41, 0x94, 0xfb, 0x87, 0x7e, 0xea, 0x0f, 0x62, - 0x31, 0xe2, 0x83, 0xeb, 0xff, 0x36, 0x07, 0x2d, 0xf3, 0xb3, 0xfa, 0x3b, 0x00, 0x00, 0xff, 0xff, - 0xa9, 0x7c, 0xb5, 0x30, 0xbf, 0x04, 0x00, 0x00, + 0x43, 0x3b, 0x74, 0x27, 0xdb, 0x76, 0x9a, 0xce, 0x2c, 0xe0, 0xcd, 0x4f, 0x60, 0x8c, 0x9f, 0xc2, + 0xab, 0x1f, 0xc1, 0x9b, 0x47, 0x8e, 0x1c, 0xa5, 0x5c, 0x3c, 0xf2, 0x11, 0xcc, 0x4c, 0xdb, 0x85, + 0x2e, 0x31, 0x7a, 0xda, 0x79, 0xbf, 0xf7, 0x7b, 0xbf, 0xf9, 0xcd, 0x7b, 0x7d, 0x0b, 0xf7, 0xd2, + 0x51, 0x38, 0x88, 0x64, 0x98, 0x66, 0x52, 0xcb, 0x81, 0x1a, 0x71, 0xed, 0x0f, 0xfb, 0x36, 0xc0, + 0xed, 0x0a, 0xee, 0x2e, 0xd5, 0x48, 0xd5, 0xa1, 0xa0, 0x79, 0xaf, 0x60, 0xf1, 0xcd, 0x98, 0x25, + 0x5a, 0x44, 0x7c, 0xd7, 0x96, 0x6f, 0x33, 0x9d, 0x89, 0x13, 0xbc, 0x06, 0xad, 0x23, 0x16, 0x8d, + 0xb9, 0x72, 0x51, 0xaf, 0xb1, 0xdc, 0x59, 0x21, 0xfd, 0x49, 0x61, 0x9d, 0xff, 0x96, 0xfb, 0x5a, + 0x66, 0xb4, 0x64, 0x7b, 0x3b, 0xd3, 0x7a, 0x45, 0x1e, 0xaf, 0xc3, 0x8c, 0x62, 0x71, 0x1a, 0xfd, + 0x5d, 0x70, 0xd7, 0xd2, 0x68, 0x45, 0xf7, 0x3e, 0xa1, 0x69, 0xc9, 0x82, 0x81, 0x1f, 0x01, 0x3a, + 0x74, 0x51, 0x0f, 0x2d, 0x77, 0x56, 0xdc, 0x3f, 0x89, 0x51, 0x74, 0x88, 0x1f, 0xc0, 0x9c, 0x16, + 0x31, 0x57, 0x9a, 0xc5, 0xe9, 0xfb, 0x58, 0xb9, 0xff, 0xf5, 0xd0, 0x72, 0x83, 0x76, 0x26, 0xd8, + 0xb6, 0xc2, 0x4f, 0xa1, 0x15, 0x73, 0x9d, 0x09, 0xdf, 0x6d, 0x58, 0x73, 0x77, 0xae, 0xf4, 0x5e, + 0xb2, 0x03, 0x1e, 0xed, 0x30, 0x91, 0xd1, 0x92, 0xe2, 0x85, 0xb0, 0x50, 0xbf, 0x04, 0x3f, 0x83, + 0x19, 0x1d, 0x88, 0x90, 0x2b, 0x5d, 0xfa, 0xb9, 0x7d, 0x55, 0xbf, 0xf7, 0xdc, 0x26, 0xb6, 0x1c, + 0x5a, 0x71, 0xf0, 0x7d, 0x68, 0x07, 0x41, 0x31, 0x2c, 0x6b, 0x66, 0x6e, 0xcb, 0xa1, 0x13, 0x64, + 0xa3, 0x0d, 0xad, 0xe2, 0xe4, 0x7d, 0x47, 0x30, 0x53, 0x96, 0xe3, 0x5b, 0xd0, 0x88, 0x45, 0x62, + 0xe5, 0x11, 0x35, 0x47, 0x8b, 0xb0, 0x13, 0x2b, 0x60, 0x10, 0x76, 0x82, 0x7b, 0xd0, 0xf1, 0x65, + 0x9c, 0x66, 0x5c, 0x29, 0x21, 0x13, 0xb7, 0x61, 0x33, 0xd7, 0x21, 0xbc, 0x0e, 0xb3, 0x69, 0x26, + 0x7d, 0xae, 0x14, 0x0f, 0xdc, 0xa6, 0x7d, 0x6a, 0xf7, 0x86, 0xd5, 0xfe, 0x26, 0x4f, 0x74, 0x26, + 0x45, 0x40, 0xaf, 0xc8, 0xdd, 0x35, 0x68, 0x57, 0x30, 0xc6, 0xd0, 0x8c, 0x39, 0xab, 0xcc, 0xd8, + 0x33, 0xbe, 0x0b, 0xad, 0x63, 0x2e, 0xc2, 0xa1, 0x2e, 0x0d, 0x95, 0x91, 0xf7, 0x0e, 0x16, 0x36, + 0xe5, 0x38, 0xd1, 0xdb, 0x22, 0x29, 0x9b, 0xb5, 0x08, 0xff, 0x07, 0x3c, 0xd5, 0x43, 0x5b, 0x3e, + 0x4f, 0x8b, 0xc0, 0xa0, 0xc7, 0x22, 0xd0, 0x45, 0x43, 0xe6, 0x69, 0x11, 0xe0, 0x2e, 0xb4, 0x7d, + 0x53, 0xcd, 0x33, 0x65, 0x27, 0x33, 0x4f, 0x27, 0xb1, 0xf7, 0x0d, 0x41, 0x73, 0x4f, 0xa6, 0x2f, + 0xf0, 0x13, 0x68, 0xf8, 0xb1, 0xba, 0xf9, 0x25, 0xd4, 0xef, 0xa5, 0x86, 0x84, 0x1f, 0x43, 0x33, + 0x12, 0xca, 0x98, 0x9c, 0x1a, 0xb3, 0x51, 0xea, 0xdb, 0x31, 0x5b, 0x82, 0xe9, 0xe5, 0xf0, 0x43, + 0xca, 0xb3, 0x48, 0x86, 0x91, 0x0c, 0x6d, 0x2f, 0xe7, 0xe8, 0x75, 0xa8, 0xbb, 0x02, 0x4d, 0xc3, + 0x37, 0xce, 0xf9, 0x11, 0x4f, 0x8a, 0xd1, 0xcf, 0xd2, 0x22, 0x30, 0xa8, 0x75, 0x5a, 0xbd, 0xc7, + 0x06, 0xde, 0x17, 0x04, 0x60, 0x6e, 0x2a, 0x97, 0x6c, 0x75, 0x6a, 0xc9, 0x96, 0xea, 0x7e, 0x0a, + 0x56, 0xbf, 0xbe, 0x61, 0xdd, 0xd7, 0xd0, 0x2a, 0x77, 0xca, 0x83, 0xa6, 0x96, 0xe9, 0xa8, 0x7c, + 0xf9, 0x42, 0xbd, 0x98, 0xda, 0xdc, 0x3f, 0x7c, 0xfc, 0x1b, 0xfb, 0xa7, 0xe7, 0xc4, 0x39, 0x3b, + 0x27, 0xce, 0xe5, 0x39, 0x41, 0x1f, 0x73, 0x82, 0xbe, 0xe6, 0x04, 0xfd, 0xc8, 0x09, 0x3a, 0xcd, + 0x09, 0xfa, 0x99, 0x13, 0xf4, 0x2b, 0x27, 0xce, 0x65, 0x4e, 0xd0, 0xe7, 0x0b, 0xe2, 0x9c, 0x5e, + 0x10, 0xe7, 0xec, 0x82, 0x38, 0xfb, 0x0f, 0x43, 0xa1, 0x87, 0xe3, 0x83, 0xbe, 0x2f, 0xe3, 0x41, + 0x98, 0xb1, 0x43, 0x96, 0xb0, 0x41, 0x24, 0x47, 0x62, 0x70, 0xfd, 0xdf, 0xe6, 0xa0, 0x65, 0x7f, + 0x56, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x24, 0x9c, 0x74, 0xb7, 0xa9, 0x04, 0x00, 0x00, } func (this *QuantileSketchMatrix) Equal(that interface{}) bool { diff --git a/pkg/logproto/sketch.proto b/pkg/logproto/sketch.proto index e84deaf20d..d8ffeb0110 100644 --- a/pkg/logproto/sketch.proto +++ b/pkg/logproto/sketch.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package logproto; -import "gogoproto/gogo.proto"; import "pkg/logproto/logproto.proto"; option go_package = "github.com/grafana/loki/pkg/logproto"; diff --git a/pkg/logql/blocker.go b/pkg/logql/blocker.go index cbfdc6bf49..9a07113c40 100644 --- a/pkg/logql/blocker.go +++ b/pkg/logql/blocker.go @@ -33,8 +33,8 @@ func (qb *queryBlocker) isBlocked(ctx context.Context, tenant string) bool { return false } - query := qb.q.params.Query() - typ, err := QueryType(query) + query := qb.q.params.QueryString() + typ, err := QueryType(qb.q.params.GetExpression()) if err != nil { typ = "unknown" } diff --git a/pkg/logql/blocker_test.go b/pkg/logql/blocker_test.go index e0dc00bf62..9fa586a02d 100644 --- a/pkg/logql/blocker_test.go +++ b/pkg/logql/blocker_test.go @@ -145,15 +145,10 @@ func TestEngine_ExecWithBlockedQueries(t *testing.T) { t.Run(test.name, func(t *testing.T) { limits.blockedQueries = test.blocked - q := eng.Query(LiteralParams{ - qs: test.q, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: logproto.FORWARD, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.q, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, logproto.FORWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectedErr == nil { require.NoError(t, err) diff --git a/pkg/logql/downstream.go b/pkg/logql/downstream.go index 3944b4fc49..2cd706c812 100644 --- a/pkg/logql/downstream.go +++ b/pkg/logql/downstream.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/grafana/loki/pkg/iter" + "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/metadata" @@ -62,15 +63,12 @@ func NewDownstreamEngine(opts EngineOpts, downstreamable Downstreamable, limits func (ng *DownstreamEngine) Opts() EngineOpts { return ng.opts } // Query constructs a Query -func (ng *DownstreamEngine) Query(ctx context.Context, p Params, mapped syntax.Expr) Query { +func (ng *DownstreamEngine) Query(ctx context.Context, p Params) Query { return &query{ logger: ng.logger, params: p, evaluator: NewDownstreamEvaluator(ng.downstreamable.Downstreamer(ctx)), - parse: func(_ context.Context, _ string) (syntax.Expr, error) { - return mapped, nil - }, - limits: ng.limits, + limits: ng.limits, } } @@ -189,9 +187,7 @@ type Downstreamable interface { } type DownstreamQuery struct { - Expr syntax.Expr Params Params - Shards Shards } // Downstreamer is an interface for deferring responsibility for query execution. @@ -268,9 +264,10 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( shards = append(shards, *e.shard) } results, err := ev.Downstream(ctx, []DownstreamQuery{{ - Expr: e.SampleExpr, - Params: params, - Shards: shards, + Params: ParamsWithShardsOverride{ + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.SampleExpr}, + ShardsOverride: Shards(shards).Encode(), + }, }}) if err != nil { return nil, err @@ -282,11 +279,10 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( var queries []DownstreamQuery for cur != nil { qry := DownstreamQuery{ - Expr: cur.DownstreamSampleExpr.SampleExpr, - Params: params, + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: cur.DownstreamSampleExpr.SampleExpr}, } if shard := cur.DownstreamSampleExpr.shard; shard != nil { - qry.Shards = Shards{*shard} + qry.Params = ParamsWithShardsOverride{Params: qry.Params, ShardsOverride: Shards{*shard}.Encode()} } queries = append(queries, qry) cur = cur.next @@ -304,7 +300,7 @@ func (ev *DownstreamEvaluator) NewStepEvaluator( level.Warn(util_log.Logger).Log( "msg", "could not extract StepEvaluator", "err", err, - "expr", queries[i].Expr.String(), + "expr", queries[i].Params.GetExpression().String(), ) return nil, err } @@ -332,25 +328,25 @@ func (ev *DownstreamEvaluator) NewIterator( shards = append(shards, *e.shard) } results, err := ev.Downstream(ctx, []DownstreamQuery{{ - Expr: e.LogSelectorExpr, - Params: params, - Shards: shards, + Params: ParamsWithShardsOverride{ + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: e.LogSelectorExpr}, + ShardsOverride: shards.Encode(), + }, }}) if err != nil { return nil, err } - return ResultIterator(results[0], params) + return ResultIterator(results[0], params.Direction()) case *ConcatLogSelectorExpr: cur := e var queries []DownstreamQuery for cur != nil { qry := DownstreamQuery{ - Expr: cur.DownstreamLogSelectorExpr.LogSelectorExpr, - Params: params, + Params: ParamsWithExpressionOverride{Params: params, ExpressionOverride: cur.DownstreamLogSelectorExpr.LogSelectorExpr}, } if shard := cur.DownstreamLogSelectorExpr.shard; shard != nil { - qry.Shards = Shards{*shard} + qry.Params = ParamsWithShardsOverride{Params: qry.Params, ShardsOverride: Shards{*shard}.Encode()} } queries = append(queries, qry) cur = cur.next @@ -363,12 +359,12 @@ func (ev *DownstreamEvaluator) NewIterator( xs := make([]iter.EntryIterator, 0, len(results)) for i, res := range results { - iter, err := ResultIterator(res, params) + iter, err := ResultIterator(res, params.Direction()) if err != nil { level.Warn(util_log.Logger).Log( "msg", "could not extract Iterator", "err", err, - "expr", queries[i].Expr.String(), + "expr", queries[i].Params.GetExpression().String(), ) } xs = append(xs, iter) @@ -452,10 +448,10 @@ func NewResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator } // ResultIterator coerces a downstream streams result into an iter.EntryIterator -func ResultIterator(res logqlmodel.Result, params Params) (iter.EntryIterator, error) { +func ResultIterator(res logqlmodel.Result, direction logproto.Direction) (iter.EntryIterator, error) { streams, ok := res.Data.(logqlmodel.Streams) if !ok { return nil, fmt.Errorf("unexpected type (%s) for ResultIterator; expected %s", res.Data.Type(), logqlmodel.ValueTypeStreams) } - return iter.NewStreamsIterator(streams, params.Direction()), nil + return iter.NewStreamsIterator(streams, direction), nil } diff --git a/pkg/logql/downstream_test.go b/pkg/logql/downstream_test.go index c5f54b9e1c..0f4d1cd099 100644 --- a/pkg/logql/downstream_test.go +++ b/pkg/logql/downstream_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" ) var nilShardMetrics = NewShardMapperMetrics(nil) @@ -69,7 +70,7 @@ func TestMappingEquivalence(t *testing.T) { sharded := NewDownstreamEngine(opts, MockDownstreamer{regular}, NoLimits, log.NewNopLogger()) t.Run(tc.query, func(t *testing.T) { - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -79,14 +80,16 @@ func TestMappingEquivalence(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) + qry := regular.Query(params) ctx := user.InjectOrgID(context.Background(), "fake") mapper := NewShardMapper(ConstantShards(shards), nilShardMetrics) - _, _, mapped, err := mapper.Parse(tc.query) + _, _, mapped, err := mapper.Parse(params.GetExpression()) require.Nil(t, err) - shardedQry := sharded.Query(ctx, params, mapped) + shardedQry := sharded.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: mapped}) res, err := qry.Exec(ctx) require.Nil(t, err) @@ -135,7 +138,7 @@ func TestShardCounter(t *testing.T) { sharded := NewDownstreamEngine(opts, MockDownstreamer{regular}, NoLimits, log.NewNopLogger()) t.Run(tc.query, func(t *testing.T) { - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -145,13 +148,14 @@ func TestShardCounter(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) ctx := user.InjectOrgID(context.Background(), "fake") mapper := NewShardMapper(ConstantShards(shards), nilShardMetrics) - noop, _, mapped, err := mapper.Parse(tc.query) - require.Nil(t, err) + noop, _, mapped, err := mapper.Parse(params.GetExpression()) + require.NoError(t, err) - shardedQry := sharded.Query(ctx, params, mapped) + shardedQry := sharded.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: mapped}) shardedRes, err := shardedQry.Exec(ctx) require.Nil(t, err) @@ -393,7 +397,7 @@ func TestRangeMappingEquivalence(t *testing.T) { t.Run(tc.query, func(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "fake") - params := NewLiteralParams( + params, err := NewLiteralParams( tc.query, start, end, @@ -403,6 +407,7 @@ func TestRangeMappingEquivalence(t *testing.T) { uint32(limit), nil, ) + require.NoError(t, err) // Regular engine qry := regularEngine.Query(params) @@ -412,12 +417,12 @@ func TestRangeMappingEquivalence(t *testing.T) { // Downstream engine - split by range rangeMapper, err := NewRangeMapper(tc.splitByInterval, nilRangeMetrics, NewMapperStats()) require.Nil(t, err) - noop, rangeExpr, err := rangeMapper.Parse(tc.query) + noop, rangeExpr, err := rangeMapper.Parse(syntax.MustParseExpr(tc.query)) require.Nil(t, err) require.False(t, noop, "downstream engine cannot execute noop") - rangeQry := downstreamEngine.Query(ctx, params, rangeExpr) + rangeQry := downstreamEngine.Query(ctx, ParamsWithExpressionOverride{Params: params, ExpressionOverride: rangeExpr}) rangeRes, err := rangeQry.Exec(ctx) require.Nil(t, err) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index af680a33b9..e04cf1dcff 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -160,12 +160,9 @@ func NewEngine(opts EngineOpts, q Querier, l Limits, logger log.Logger) *Engine // Query creates a new LogQL query. Instant/Range type is derived from the parameters. func (ng *Engine) Query(params Params) Query { return &query{ - logger: ng.logger, - params: params, - evaluator: ng.evaluatorFactory, - parse: func(_ context.Context, query string) (syntax.Expr, error) { - return syntax.ParseExpr(query) - }, + logger: ng.logger, + params: params, + evaluator: ng.evaluatorFactory, record: true, logExecQuery: ng.opts.LogExecutingQuery, limits: ng.limits, @@ -181,7 +178,6 @@ type Query interface { type query struct { logger log.Logger params Params - parse func(context.Context, string) (syntax.Expr, error) limits Limits evaluator EvaluatorFactory record bool @@ -211,7 +207,7 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { sp.LogKV( "type", GetRangeType(q.params), - "query", q.params.Query(), + "query", q.params.QueryString(), "start", q.params.Start(), "end", q.params.End(), "step", q.params.Step(), @@ -219,11 +215,11 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) { ) if q.logExecQuery { - queryHash := util.HashedQuery(q.params.Query()) + queryHash := util.HashedQuery(q.params.QueryString()) if GetRangeType(q.params) == InstantType { - level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "instant", "query", q.params.Query(), "query_hash", queryHash) + level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "instant", "query", q.params.QueryString(), "query_hash", queryHash) } else { - level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "range", "query", q.params.Query(), "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step(), "query_hash", queryHash) + level.Info(logutil.WithContext(ctx, q.logger)).Log("msg", "executing query", "type", "range", "query", q.params.QueryString(), "length", q.params.End().Sub(q.params.Start()), "step", q.params.Step(), "query_hash", queryHash) } } @@ -263,16 +259,11 @@ func (q *query) Eval(ctx context.Context) (promql_parser.Value, error) { ctx, cancel := context.WithTimeout(ctx, queryTimeout) defer cancel() - expr, err := q.parse(ctx, q.params.Query()) - if err != nil { - return nil, err - } - if q.checkBlocked(ctx, tenants) { return nil, logqlmodel.ErrBlocked } - switch e := expr.(type) { + switch e := q.params.GetExpression().(type) { case syntax.SampleExpr: value, err := q.evalSample(ctx, e) return value, err @@ -364,7 +355,7 @@ func (q *query) evalSample(ctx context.Context, expr syntax.SampleExpr) (promql_ if GetRangeType(q.params) == InstantType { sortByValue, err := Sortable(q.params) if err != nil { - return nil, fmt.Errorf("fail to check Sortable, logql: %s ,err: %s", q.params.Query(), err) + return nil, fmt.Errorf("fail to check Sortable, logql: %s ,err: %s", q.params.QueryString(), err) } if !sortByValue { sort.Slice(vec, func(i, j int) bool { return labels.Compare(vec[i].Metric, vec[j].Metric) < 0 }) diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index 548400644a..e0b6ab3dff 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -129,13 +129,9 @@ func TestEngine_LogsRateUnwrap(t *testing.T) { t.Parallel() eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.ts, - end: test.ts, - direction: test.direction, - limit: test.limit, - }) + params, err := NewLiteralParams(test.qs, test.ts, test.ts, 0, 0, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if expectedError, ok := test.expected.(error); ok { assert.Equal(t, expectedError.Error(), err.Error()) @@ -960,13 +956,10 @@ func TestEngine_LogsInstantQuery(t *testing.T) { t.Parallel() eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.ts, - end: test.ts, - direction: test.direction, - limit: test.limit, - }) + + params, err := NewLiteralParams(test.qs, test.ts, test.ts, 0, 0, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if expectedError, ok := test.expected.(error); ok { assert.Equal(t, expectedError.Error(), err.Error()) @@ -2266,15 +2259,9 @@ func TestEngine_RangeQuery(t *testing.T) { eng := NewEngine(EngineOpts{}, newQuerierRecorder(t, test.data, test.params), NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: test.qs, - start: test.start, - end: test.end, - step: test.step, - interval: test.interval, - direction: test.direction, - limit: test.limit, - }) + params, err := NewLiteralParams(test.qs, test.start, test.end, test.step, test.interval, test.direction, test.limit, nil) + require.NoError(t, err) + q := eng.Query(params) res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if err != nil { t.Fatal(err) @@ -2302,13 +2289,11 @@ func TestEngine_Stats(t *testing.T) { eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) queueTime := 2 * time.Nanosecond - q := eng.Query(LiteralParams{ - qs: `{foo="bar"}`, - start: time.Now(), - end: time.Now(), - direction: logproto.BACKWARD, - limit: 1000, - }) + + params, err := NewLiteralParams(`{foo="bar"}`, time.Now(), time.Now(), 0, 0, logproto.FORWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) r, err := q.Exec(user.InjectOrgID(ctx, "fake")) require.NoError(t, err) @@ -2338,13 +2323,9 @@ func (metaQuerier) SelectSamples(ctx context.Context, _ SelectSampleParams) (ite func TestEngine_Metadata(t *testing.T) { eng := NewEngine(EngineOpts{}, &metaQuerier{}, NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: `{foo="bar"}`, - start: time.Now(), - end: time.Now(), - direction: logproto.BACKWARD, - limit: 1000, - }) + params, err := NewLiteralParams(`{foo="bar"}`, time.Now(), time.Now(), 0, 0, logproto.BACKWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) r, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) require.NoError(t, err) @@ -2353,51 +2334,17 @@ func TestEngine_Metadata(t *testing.T) { }, r.Headers) } -func TestEngine_LogsInstantQuery_IllegalLogql(t *testing.T) { - eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) - - queueTime := 2 * time.Nanosecond - illegalVector := `vector(abc)` - q := eng.Query(LiteralParams{ - qs: illegalVector, - start: time.Now(), - end: time.Now(), - step: time.Second * 30, - interval: time.Second * 30, - direction: logproto.BACKWARD, - limit: 1000, - }) - expectErr := logqlmodel.NewParseError("syntax error: unexpected IDENTIFIER, expecting NUMBER", 1, 8) - ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) - _, err := q.Exec(user.InjectOrgID(ctx, "fake")) - - require.EqualError(t, err, expectErr.Error()) - - qry, ok := q.(*query) - require.Equal(t, ok, true) - vectorExpr := syntax.NewVectorExpr(illegalVector) - - _, err = qry.evalSample(ctx, vectorExpr) - expectEvalSampleErr := logqlmodel.NewParseError("unable to parse vectorExpr as a float: strconv.ParseFloat: parsing \"vector(abc)\": invalid syntax", 0, 0) - require.EqualError(t, err, expectEvalSampleErr.Error()) -} - func TestEngine_LogsInstantQuery_Vector(t *testing.T) { eng := NewEngine(EngineOpts{}, &statsQuerier{}, NoLimits, log.NewNopLogger()) now := time.Now() queueTime := 2 * time.Nanosecond logqlVector := `vector(5)` - q := eng.Query(LiteralParams{ - qs: logqlVector, - start: now, - end: now, - step: 0, - interval: time.Second * 30, - direction: logproto.BACKWARD, - limit: 1000, - }) + + params, err := NewLiteralParams(logqlVector, now, now, 0, time.Second*30, logproto.BACKWARD, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) ctx := context.WithValue(context.Background(), httpreq.QueryQueueTimeHTTPHeader, queueTime) - _, err := q.Exec(user.InjectOrgID(ctx, "fake")) + _, err = q.Exec(user.InjectOrgID(ctx, "fake")) require.NoError(t, err) @@ -2472,14 +2419,11 @@ func TestStepEvaluator_Error(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { eng := NewEngine(EngineOpts{}, tc.querier, NoLimits, log.NewNopLogger()) - q := eng.Query(LiteralParams{ - qs: tc.qs, - start: time.Unix(0, 0), - end: time.Unix(180, 0), - step: 1 * time.Second, - limit: 1, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + + params, err := NewLiteralParams(tc.qs, time.Unix(0, 0), time.Unix(180, 0), 1*time.Second, 0, logproto.BACKWARD, 1, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) require.Equal(t, tc.err, err) }) } @@ -2502,15 +2446,10 @@ func TestEngine_MaxSeries(t *testing.T) { {`avg(count_over_time({app=~"foo|bar"} |~".+bar" [1m]))`, logproto.FORWARD, false}, } { t.Run(test.qs, func(t *testing.T) { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.qs, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, test.direction, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectLimitErr { require.NotNil(t, err) require.True(t, errors.Is(err, logqlmodel.ErrLimit)) @@ -2534,15 +2473,11 @@ func TestEngine_MaxRangeInterval(t *testing.T) { {`topk(1,rate({app=~"foo|bar"}[12h]) / (rate({app="baz"}[23h]) + rate({app="fiz"}[25h])))`, logproto.FORWARD, true}, } { t.Run(test.qs, func(t *testing.T) { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: time.Unix(0, 0), - end: time.Unix(100000, 0), - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) - _, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) + params, err := NewLiteralParams(test.qs, time.Unix(0, 0), time.Unix(100000, 0), 60*time.Second, 0, test.direction, 1000, nil) + require.NoError(t, err) + q := eng.Query(params) + + _, err = q.Exec(user.InjectOrgID(context.Background(), "fake")) if test.expectLimitErr { require.Error(t, err) require.ErrorIs(t, err, logqlmodel.ErrIntervalLimit) @@ -2605,14 +2540,10 @@ func benchmarkRangeQuery(testsize int64, b *testing.B) { {`bottomk(2,rate(({app=~"foo|bar"} |~".+bar")[1m]))`, logproto.FORWARD}, {`bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app)`, logproto.FORWARD}, } { - q := eng.Query(LiteralParams{ - qs: test.qs, - start: start, - end: end, - step: 60 * time.Second, - direction: test.direction, - limit: 1000, - }) + params, err := NewLiteralParams(test.qs, start, end, 60*time.Second, 0, logproto.BACKWARD, 1000, nil) + require.NoError(b, err) + q := eng.Query(params) + res, err := q.Exec(user.InjectOrgID(context.Background(), "fake")) if err != nil { b.Fatal(err) @@ -2640,8 +2571,13 @@ func TestHashingStability(t *testing.T) { buf := bytes.NewBufferString("") logger := log.NewLogfmtLogger(buf) eng := NewEngine(EngineOpts{LogExecutingQuery: true}, getLocalQuerier(4), NoLimits, logger) + + parsed, err := syntax.ParseExpr(params.QueryString()) + require.NoError(t, err) + params.queryExpr = parsed + query := eng.Query(params) - _, err := query.Exec(ctx) + _, err = query.Exec(ctx) require.NoError(t, err) return buf.String() } @@ -2668,7 +2604,7 @@ func TestHashingStability(t *testing.T) { {`sum by(query_hash) (count_over_time({app="myapp",env="myenv"} |= "error" |= "metrics.go" | logfmt [10s]))`}, {`sum (count_over_time({app="myapp",env="myenv"} |= "error" |= "metrics.go" | logfmt [10s])) by(query_hash)`}, } { - params.qs = test.qs + params.queryString = test.qs expectedQueryHash := util.HashedQuery(test.qs) // check that both places will end up having the same query hash, even though they're emitting different log lines. diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 0c0dba2cad..fdb9190956 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -31,7 +31,7 @@ var ( // Params details the parameters associated with a loki request type Params interface { - Query() string + QueryString() string Start() time.Time End() time.Time Step() time.Duration @@ -39,6 +39,7 @@ type Params interface { Limit() uint32 Direction() logproto.Direction Shards() []string + GetExpression() syntax.Expr } func NewLiteralParams( @@ -48,33 +49,41 @@ func NewLiteralParams( direction logproto.Direction, limit uint32, shards []string, -) LiteralParams { - return LiteralParams{ - qs: qs, - start: start, - end: end, - step: step, - interval: interval, - direction: direction, - limit: limit, - shards: shards, - } +) (LiteralParams, error) { + p := LiteralParams{ + queryString: qs, + start: start, + end: end, + step: step, + interval: interval, + direction: direction, + limit: limit, + shards: shards, + } + var err error + p.queryExpr, err = syntax.ParseExpr(qs) + return p, err + } // LiteralParams impls Params type LiteralParams struct { - qs string + queryString string start, end time.Time step, interval time.Duration direction logproto.Direction limit uint32 shards []string + queryExpr syntax.Expr } func (p LiteralParams) Copy() LiteralParams { return p } // String impls Params -func (p LiteralParams) Query() string { return p.qs } +func (p LiteralParams) QueryString() string { return p.queryString } + +// GetExpression impls Params +func (p LiteralParams) GetExpression() syntax.Expr { return p.queryExpr } // Start impls Params func (p LiteralParams) Start() time.Time { return p.start } @@ -105,12 +114,38 @@ func GetRangeType(q Params) QueryRangeType { return RangeType } +// ParamsWithExpressionOverride overrides the query expression so that the query +// string and the expression can differ. This is useful for for query planning +// when plan my not match externally available logql syntax +type ParamsWithExpressionOverride struct { + Params + ExpressionOverride syntax.Expr +} + +// GetExpression returns the parsed expression of the query. +func (p ParamsWithExpressionOverride) GetExpression() syntax.Expr { + return p.ExpressionOverride +} + +// ParamsWithExpressionOverride overrides the shards. Since the backing +// implementation of the Params interface is unknown they are embedded and the +// original shards are shadowed. +type ParamsWithShardsOverride struct { + Params + ShardsOverride []string +} + +// Shards returns this overwriting shards. +func (p ParamsWithShardsOverride) Shards() []string { + return p.ShardsOverride +} + // Sortable logql contain sort or sort_desc. func Sortable(q Params) (bool, error) { var sortable bool - expr, err := syntax.ParseSampleExpr(q.Query()) - if err != nil { - return false, err + expr, ok := q.GetExpression().(syntax.SampleExpr) + if !ok { + return false, errors.New("only sample expression supported") } expr.Walk(func(e syntax.Expr) { rangeExpr, ok := e.(*syntax.VectorAggregationExpr) diff --git a/pkg/logql/evaluator_test.go b/pkg/logql/evaluator_test.go index 1bec3d9c67..e31d587252 100644 --- a/pkg/logql/evaluator_test.go +++ b/pkg/logql/evaluator_test.go @@ -44,14 +44,14 @@ func TestDefaultEvaluator_DivideByZero(t *testing.T) { } func TestDefaultEvaluator_Sortable(t *testing.T) { logqlSort := `sort(rate(({app=~"foo|bar"} |~".+bar")[1m])) ` - sortable, err := Sortable(LiteralParams{qs: logqlSort}) + sortable, err := Sortable(LiteralParams{queryString: logqlSort, queryExpr: syntax.MustParseExpr(logqlSort)}) if err != nil { t.Fatal(err) } require.Equal(t, true, sortable) logqlSum := `sum(rate(({app=~"foo|bar"} |~".+bar")[1m])) ` - sortableSum, err := Sortable(LiteralParams{qs: logqlSum}) + sortableSum, err := Sortable(LiteralParams{queryString: logqlSum, queryExpr: syntax.MustParseExpr(logqlSum)}) if err != nil { t.Fatal(err) } diff --git a/pkg/logql/explain_test.go b/pkg/logql/explain_test.go index a54ffa5916..5ae2f840e1 100644 --- a/pkg/logql/explain_test.go +++ b/pkg/logql/explain_test.go @@ -29,14 +29,14 @@ func TestExplain(t *testing.T) { downEv := &DownstreamEvaluator{Downstreamer: MockDownstreamer{regular}, defaultEvaluator: defaultEv} mapper := NewShardMapper(ConstantShards(4), nilShardMetrics) - _, _, expr, err := mapper.Parse(query) + _, _, expr, err := mapper.Parse(syntax.MustParseExpr(query)) require.NoError(t, err) params := LiteralParams{ - qs: query, - start: time.Unix(60, 0), - end: time.Unix(60, 0), - limit: 1000, + queryString: query, + start: time.Unix(60, 0), + end: time.Unix(60, 0), + limit: 1000, } ev, err := downEv.NewStepEvaluator(ctx, downEv, expr.(syntax.SampleExpr), params) diff --git a/pkg/logql/metrics.go b/pkg/logql/metrics.go index 94a4c2f9dd..9db8ee96e4 100644 --- a/pkg/logql/metrics.go +++ b/pkg/logql/metrics.go @@ -98,7 +98,7 @@ func RecordRangeAndInstantQueryMetrics( latencyType = latencyTypeFast returnedLines = 0 ) - queryType, err := QueryType(p.Query()) + queryType, err := QueryType(p.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "error parsing query type", "err", err) } @@ -119,8 +119,8 @@ func RecordRangeAndInstantQueryMetrics( logValues = append(logValues, []interface{}{ "latency", latencyType, // this can be used to filter log lines. - "query", p.Query(), - "query_hash", util.HashedQuery(p.Query()), + "query", p.QueryString(), + "query_hash", util.HashedQuery(p.QueryString()), "query_type", queryType, "range_type", rt, "length", p.End().Sub(p.Start()), @@ -373,11 +373,7 @@ func recordUsageStats(queryType string, stats logql_stats.Result) { } } -func QueryType(query string) (string, error) { - expr, err := syntax.ParseExpr(query) - if err != nil { - return "", err - } +func QueryType(expr syntax.Expr) (string, error) { switch e := expr.(type) { case syntax.SampleExpr: return QueryTypeMetric, nil diff --git a/pkg/logql/metrics_test.go b/pkg/logql/metrics_test.go index 06d4e26994..6d07040bb8 100644 --- a/pkg/logql/metrics_test.go +++ b/pkg/logql/metrics_test.go @@ -16,6 +16,7 @@ import ( "github.com/uber/jaeger-client-go" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" "github.com/grafana/loki/pkg/util" @@ -25,30 +26,25 @@ import ( func TestQueryType(t *testing.T) { tests := []struct { - name string - query string - want string - wantErr bool + name string + query string + want string }{ - {"bad", "ddd", "", true}, - {"limited", `{app="foo"}`, QueryTypeLimited, false}, - {"limited multi label", `{app="foo" ,fuzz=~"foo"}`, QueryTypeLimited, false}, - {"limited with parser", `{app="foo" ,fuzz=~"foo"} | logfmt`, QueryTypeLimited, false}, - {"filter", `{app="foo"} |= "foo"`, QueryTypeFilter, false}, - {"filter string extracted label", `{app="foo"} | json | foo="a"`, QueryTypeFilter, false}, - {"filter duration", `{app="foo"} | json | duration > 5s`, QueryTypeFilter, false}, - {"metrics", `rate({app="foo"} |= "foo"[5m])`, QueryTypeMetric, false}, - {"metrics binary", `rate({app="foo"} |= "foo"[5m]) + count_over_time({app="foo"} |= "foo"[5m]) / rate({app="foo"} |= "foo"[5m]) `, QueryTypeMetric, false}, - {"filters", `{app="foo"} |= "foo" |= "f" != "b"`, QueryTypeFilter, false}, - {"filters and labels filters", `{app="foo"} |= "foo" |= "f" != "b" | json | a > 5`, QueryTypeFilter, false}, + {"limited", `{app="foo"}`, QueryTypeLimited}, + {"limited multi label", `{app="foo" ,fuzz=~"foo"}`, QueryTypeLimited}, + {"limited with parser", `{app="foo" ,fuzz=~"foo"} | logfmt`, QueryTypeLimited}, + {"filter", `{app="foo"} |= "foo"`, QueryTypeFilter}, + {"filter string extracted label", `{app="foo"} | json | foo="a"`, QueryTypeFilter}, + {"filter duration", `{app="foo"} | json | duration > 5s`, QueryTypeFilter}, + {"metrics", `rate({app="foo"} |= "foo"[5m])`, QueryTypeMetric}, + {"metrics binary", `rate({app="foo"} |= "foo"[5m]) + count_over_time({app="foo"} |= "foo"[5m]) / rate({app="foo"} |= "foo"[5m]) `, QueryTypeMetric}, + {"filters", `{app="foo"} |= "foo" |= "f" != "b"`, QueryTypeFilter}, + {"filters and labels filters", `{app="foo"} |= "foo" |= "f" != "b" | json | a > 5`, QueryTypeFilter}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := QueryType(tt.query) - if (err != nil) != tt.wantErr { - t.Errorf("QueryType() error = %v, wantErr %v", err, tt.wantErr) - return - } + got, err := QueryType(syntax.MustParseExpr(tt.query)) + require.NoError(t, err) if got != tt.want { t.Errorf("QueryType() = %v, want %v", got, tt.want) } @@ -69,12 +65,13 @@ func TestLogSlowQuery(t *testing.T) { ctx = context.WithValue(ctx, httpreq.QueryTagsHTTPHeader, "Source=logvolhist,Feature=Beta") RecordRangeAndInstantQueryMetrics(ctx, util_log.Logger, LiteralParams{ - qs: `{foo="bar"} |= "buzz"`, - direction: logproto.BACKWARD, - end: now, - start: now.Add(-1 * time.Hour), - limit: 1000, - step: time.Minute, + queryString: `{foo="bar"} |= "buzz"`, + direction: logproto.BACKWARD, + end: now, + start: now.Add(-1 * time.Hour), + limit: 1000, + step: time.Minute, + queryExpr: syntax.MustParseExpr(`{foo="bar"} |= "buzz"`), }, "200", stats.Result{ Summary: stats.Summary{ BytesProcessedPerSecond: 100000, diff --git a/pkg/logql/rangemapper.go b/pkg/logql/rangemapper.go index cc63944bc0..250f586603 100644 --- a/pkg/logql/rangemapper.go +++ b/pkg/logql/rangemapper.go @@ -81,10 +81,10 @@ func NewRangeMapperMetrics(registerer prometheus.Registerer) *MapperMetrics { // be executed by the downstream engine. // It returns a boolean indicating whether a rewrite was possible, the // rewritten sample expression, and an error in case the rewrite failed. -func (m RangeMapper) Parse(query string) (bool, syntax.Expr, error) { - origExpr, err := syntax.ParseSampleExpr(query) - if err != nil { - return true, nil, err +func (m RangeMapper) Parse(expr syntax.Expr) (bool, syntax.Expr, error) { + origExpr, ok := expr.(syntax.SampleExpr) + if !ok { + return true, nil, errors.New("only sample expression supported") } recorder := m.metrics.downstreamRecorder() diff --git a/pkg/logql/rangemapper_test.go b/pkg/logql/rangemapper_test.go index 1c2f827867..48394d219b 100644 --- a/pkg/logql/rangemapper_test.go +++ b/pkg/logql/rangemapper_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logql/syntax" ) func Test_SplitRangeInterval(t *testing.T) { @@ -83,7 +83,7 @@ func Test_SplitRangeInterval(t *testing.T) { rvm, err := NewRangeMapper(2*time.Second, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1741,7 +1741,7 @@ func Test_SplitRangeVectorMapping(t *testing.T) { rvm, err := NewRangeMapper(time.Minute, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1932,7 +1932,7 @@ func Test_SplitRangeVectorMapping_Noop(t *testing.T) { rvm, err := NewRangeMapper(time.Minute, nilShardMetrics, mapperStats) require.NoError(t, err) - noop, mappedExpr, err := rvm.Parse(tc.expr) + noop, mappedExpr, err := rvm.Parse(syntax.MustParseExpr(tc.expr)) require.NoError(t, err) require.Equal(t, removeWhiteSpace(tc.expected), removeWhiteSpace(mappedExpr.String())) @@ -1945,21 +1945,9 @@ func Test_SplitRangeVectorMapping_Noop(t *testing.T) { func Test_FailQuery(t *testing.T) { rvm, err := NewRangeMapper(2*time.Minute, nilShardMetrics, NewMapperStats()) require.NoError(t, err) - _, _, err = rvm.Parse(`{app="foo"} |= "err"`) + _, _, err = rvm.Parse(syntax.MustParseExpr(`{app="foo"} |= "err"`)) require.Error(t, err) - _, _, err = rvm.Parse(`topk(0, sum(count_over_time({app="foo"} | json | __error__="" [15m])))`) - require.Error(t, err) - // Check fixes for bug where missing or empty parameters for regexp and pattern parsers threw a panic - // Missing parameter to regexp parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | regexp [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) - // Empty parameter to regexp parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | regexp ` + "``" + ` [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) - // Empty parameter to pattern parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | pattern ` + `""` + ` [1d])))`) - require.ErrorIs(t, err, logqlmodel.ErrParse) // Empty parameter to json parser - _, _, err = rvm.Parse(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | json [1d])))`) + _, _, err = rvm.Parse(syntax.MustParseExpr(`topk(10,sum by(namespace)(count_over_time({application="nginx", site!="eu-west-1-dev"} |= "/artifactory/" != "api" != "binarystore" | json [1d])))`)) require.NoError(t, err) } diff --git a/pkg/logql/shardmapper.go b/pkg/logql/shardmapper.go index 6409cdbf08..f1ee7e4ba6 100644 --- a/pkg/logql/shardmapper.go +++ b/pkg/logql/shardmapper.go @@ -41,12 +41,7 @@ func NewShardMapperMetrics(registerer prometheus.Registerer) *MapperMetrics { return newMapperMetrics(registerer, "shard") } -func (m ShardMapper) Parse(query string) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error) { - parsed, err := syntax.ParseExpr(query) - if err != nil { - return false, 0, nil, err - } - +func (m ShardMapper) Parse(parsed syntax.Expr) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error) { recorder := m.metrics.downstreamRecorder() mapped, bytesPerShard, err := m.Map(parsed, recorder) diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index bdfd8a6c42..80b2e68751 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -1361,14 +1361,14 @@ func mustNewMatcher(t labels.MatchType, n, v string) *labels.Matcher { func TestStringTrimming(t *testing.T) { for _, tc := range []struct { - expr string + expr syntax.Expr expected string shards int }{ { // sample expr in entirety for low shard count shards: 2, - expr: `count_over_time({app="foo"}[1m])`, + expr: syntax.MustParseExpr(`count_over_time({app="foo"}[1m])`), expected: ` downstream ++ downstream @@ -1377,7 +1377,7 @@ func TestStringTrimming(t *testing.T) { { // sample expr doesnt display infinite shards shards: 5, - expr: `count_over_time({app="foo"}[1m])`, + expr: syntax.MustParseExpr(`count_over_time({app="foo"}[1m])`), expected: ` downstream ++ downstream ++ @@ -1389,7 +1389,7 @@ func TestStringTrimming(t *testing.T) { { // log selector expr in entirety for low shard count shards: 2, - expr: `{app="foo"}`, + expr: syntax.MustParseExpr(`{app="foo"}`), expected: ` downstream<{app="foo"},shard=0_of_2> ++ downstream<{app="foo"},shard=1_of_2> @@ -1398,7 +1398,7 @@ func TestStringTrimming(t *testing.T) { { // log selector expr doesnt display infinite shards shards: 5, - expr: `{app="foo"}`, + expr: syntax.MustParseExpr(`{app="foo"}`), expected: ` downstream<{app="foo"},shard=0_of_5> ++ downstream<{app="foo"},shard=1_of_5> ++ @@ -1408,7 +1408,7 @@ func TestStringTrimming(t *testing.T) { `, }, } { - t.Run(tc.expr, func(t *testing.T) { + t.Run(tc.expr.String(), func(t *testing.T) { m := NewShardMapper(ConstantShards(tc.shards), nilShardMetrics) _, _, mappedExpr, err := m.Parse(tc.expr) require.Nil(t, err) diff --git a/pkg/logql/syntax/parser.go b/pkg/logql/syntax/parser.go index e1fe5971ff..81874ba6d6 100644 --- a/pkg/logql/syntax/parser.go +++ b/pkg/logql/syntax/parser.go @@ -99,6 +99,14 @@ func ParseExprWithoutValidation(input string) (expr Expr, err error) { return p.Parse() } +func MustParseExpr(input string) Expr { + expr, err := ParseExpr(input) + if err != nil { + panic(err) + } + return expr +} + func validateExpr(expr Expr) error { switch e := expr.(type) { case SampleExpr: diff --git a/pkg/logql/test_utils.go b/pkg/logql/test_utils.go index 982fa7f5f1..b979dedb42 100644 --- a/pkg/logql/test_utils.go +++ b/pkg/logql/test_utils.go @@ -218,17 +218,7 @@ func (m MockDownstreamer) Downstreamer(_ context.Context) Downstreamer { return func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery) ([]logqlmodel.Result, error) { results := make([]logqlmodel.Result, 0, len(queries)) for _, query := range queries { - params := NewLiteralParams( - query.Expr.String(), - query.Params.Start(), - query.Params.End(), - query.Params.Step(), - query.Params.Interval(), - query.Params.Direction(), - query.Params.Limit(), - query.Shards.Encode(), - ) - res, err := m.Query(params).Exec(ctx) + res, err := m.Query(query.Params).Exec(ctx) if err != nil { return nil, err } diff --git a/pkg/querier/plan/plan.go b/pkg/querier/plan/plan.go new file mode 100644 index 0000000000..6822932d7b --- /dev/null +++ b/pkg/querier/plan/plan.go @@ -0,0 +1,101 @@ +package plan + +import ( + "bytes" + + "github.com/grafana/loki/pkg/logql/syntax" +) + +type QueryPlan struct { + AST syntax.Expr +} + +func (t QueryPlan) Marshal() ([]byte, error) { + return t.MarshalJSON() +} + +func (t *QueryPlan) MarshalTo(data []byte) (int, error) { + appender := &appendWriter{ + slice: data[:0], + } + err := syntax.EncodeJSON(t.AST, appender) + if err != nil { + return 0, err + } + + return len(appender.slice), nil +} + +func (t *QueryPlan) Unmarshal(data []byte) error { + return t.UnmarshalJSON(data) +} + +func (t *QueryPlan) Size() int { + counter := &countWriter{} + err := syntax.EncodeJSON(t.AST, counter) + if err != nil { + return 0 + } + + return counter.bytes +} + +func (t QueryPlan) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + err := syntax.EncodeJSON(t.AST, &buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (t *QueryPlan) UnmarshalJSON(data []byte) error { + // An empty query plan is ingored to be backwards compatible. + if len(data) == 0 { + return nil + } + + expr, err := syntax.DecodeJSON(string(data)) + if err != nil { + return err + } + + t.AST = expr + return nil +} + +func (t QueryPlan) Equal(other QueryPlan) bool { + left, err := t.Marshal() + if err != nil { + return false + } + + right, err := other.Marshal() + if err != nil { + return false + } + return bytes.Equal(left, right) +} + +// countWriter is not writing any bytes. It just counts the bytes that would be +// written. +type countWriter struct { + bytes int +} + +// Write implements io.Writer. +func (w *countWriter) Write(p []byte) (int, error) { + w.bytes += len(p) + return len(p), nil +} + +// appendWriter appends to a slice. +type appendWriter struct { + slice []byte +} + +func (w *appendWriter) Write(p []byte) (int, error) { + w.slice = append(w.slice, p...) + return len(p), nil +} diff --git a/pkg/querier/plan/plan_test.go b/pkg/querier/plan/plan_test.go new file mode 100644 index 0000000000..60f7d3fad1 --- /dev/null +++ b/pkg/querier/plan/plan_test.go @@ -0,0 +1,26 @@ +package plan + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/logql/syntax" +) + +func TestMarshalTo(t *testing.T) { + plan := QueryPlan{ + AST: syntax.MustParseExpr(`sum by (foo) (bytes_over_time({app="loki"} [1m]))`), + } + + data := make([]byte, plan.Size()) + _, err := plan.MarshalTo(data) + require.NoError(t, err) + + var buf bytes.Buffer + err = syntax.EncodeJSON(plan.AST, &buf) + require.NoError(t, err) + + require.JSONEq(t, buf.String(), string(data)) +} diff --git a/pkg/querier/queryrange/codec.go b/pkg/querier/queryrange/codec.go index 2167a5134b..b0c56a7439 100644 --- a/pkg/querier/queryrange/codec.go +++ b/pkg/querier/queryrange/codec.go @@ -29,6 +29,7 @@ import ( "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" indexStats "github.com/grafana/loki/pkg/storage/stores/index/stats" "github.com/grafana/loki/pkg/util" @@ -259,6 +260,11 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + parsed, err := syntax.ParseExpr(rangeQuery.Query) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiRequest{ Query: rangeQuery.Query, Limit: rangeQuery.Limit, @@ -269,12 +275,21 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer Interval: rangeQuery.Interval.Milliseconds(), Path: r.URL.Path, Shards: rangeQuery.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, nil case InstantQueryOp: req, err := loghttp.ParseInstantQuery(r) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiInstantRequest{ Query: req.Query, Limit: req.Limit, @@ -282,6 +297,9 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer TimeTs: req.Ts.UTC(), Path: r.URL.Path, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, nil case SeriesOp: req, err := loghttp.ParseAndValidateSeriesQuery(r) @@ -409,6 +427,12 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) if err != nil { return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiRequest{ Query: req.Query, Limit: req.Limit, @@ -419,12 +443,21 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) Interval: req.Interval.Milliseconds(), Path: r.Url, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, ctx, nil case InstantQueryOp: req, err := loghttp.ParseInstantQuery(httpReq) if err != nil { return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + + parsed, err := syntax.ParseExpr(req.Query) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiInstantRequest{ Query: req.Query, Limit: req.Limit, @@ -432,6 +465,9 @@ func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) TimeTs: req.Ts.UTC(), Path: r.Url, Shards: req.Shards, + Plan: &plan.QueryPlan{ + AST: parsed, + }, }, ctx, nil case SeriesOp: req, err := loghttp.ParseAndValidateSeriesQuery(httpReq) @@ -1429,10 +1465,14 @@ type paramsRangeWrapper struct { *LokiRequest } -func (p paramsRangeWrapper) Query() string { +func (p paramsRangeWrapper) QueryString() string { return p.GetQuery() } +func (p paramsRangeWrapper) GetExpression() syntax.Expr { + return p.LokiRequest.Plan.AST +} + func (p paramsRangeWrapper) Start() time.Time { return p.GetStartTs() } @@ -1459,10 +1499,14 @@ type paramsInstantWrapper struct { *LokiInstantRequest } -func (p paramsInstantWrapper) Query() string { +func (p paramsInstantWrapper) QueryString() string { return p.GetQuery() } +func (p paramsInstantWrapper) GetExpression() syntax.Expr { + return p.LokiInstantRequest.Plan.AST +} + func (p paramsInstantWrapper) Start() time.Time { return p.LokiInstantRequest.GetTimeTs() } @@ -1487,10 +1531,14 @@ type paramsSeriesWrapper struct { *LokiSeriesRequest } -func (p paramsSeriesWrapper) Query() string { +func (p paramsSeriesWrapper) QueryString() string { return p.GetQuery() } +func (p paramsSeriesWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsSeriesWrapper) Start() time.Time { return p.LokiSeriesRequest.GetStartTs() } @@ -1515,10 +1563,14 @@ type paramsLabelWrapper struct { *LabelRequest } -func (p paramsLabelWrapper) Query() string { +func (p paramsLabelWrapper) QueryString() string { return p.GetQuery() } +func (p paramsLabelWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsLabelWrapper) Start() time.Time { return p.LabelRequest.GetStartTs() } @@ -1543,10 +1595,14 @@ type paramsStatsWrapper struct { *logproto.IndexStatsRequest } -func (p paramsStatsWrapper) Query() string { +func (p paramsStatsWrapper) QueryString() string { return p.GetQuery() } +func (p paramsStatsWrapper) GetExpression() syntax.Expr { + return nil +} + func (p paramsStatsWrapper) Start() time.Time { return p.From.Time() } diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index 77df78aca6..c722b12af7 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -25,8 +25,10 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/util" "github.com/grafana/loki/pkg/util/httpreq" @@ -63,6 +65,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/query_range", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"}`), + }, }, false}, {"query_range", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -76,6 +81,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/query_range", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"}`), + }, }, false}, {"legacy query_range with refexp", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -89,6 +97,9 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { Path: "/api/prom/query", StartTs: start, EndTs: end, + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`{foo="bar"} |~ "foo"`), + }, }, false}, {"series", func() (*http.Request, error) { return http.NewRequest(http.MethodGet, @@ -559,7 +570,13 @@ func Test_codec_DecodeProtobufResponseParity(t *testing.T) { } codec := RequestProtobufCodec{} for i, queryTest := range queryTests { - u := &url.URL{Path: "/loki/api/v1/query_range"} + params := url.Values{ + "query": []string{`{app="foo"}`}, + } + u := &url.URL{ + Path: "/loki/api/v1/query_range", + RawQuery: params.Encode(), + } httpReq := &http.Request{ Method: "GET", RequestURI: u.String(), diff --git a/pkg/querier/queryrange/downstreamer.go b/pkg/querier/queryrange/downstreamer.go index b7a3d2f57a..860aa980fb 100644 --- a/pkg/querier/queryrange/downstreamer.go +++ b/pkg/querier/queryrange/downstreamer.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/metadata" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" "github.com/grafana/loki/pkg/util/spanlogger" @@ -34,19 +35,22 @@ type DownstreamHandler struct { next queryrangebase.Handler } -func ParamsToLokiRequest(params logql.Params, shards logql.Shards) queryrangebase.Request { +func ParamsToLokiRequest(params logql.Params) queryrangebase.Request { if logql.GetRangeType(params) == logql.InstantType { return &LokiInstantRequest{ - Query: params.Query(), + Query: params.QueryString(), Limit: params.Limit(), TimeTs: params.Start(), Direction: params.Direction(), Path: "/loki/api/v1/query", // TODO(owen-d): make this derivable - Shards: shards.Encode(), + Shards: params.Shards(), + Plan: &plan.QueryPlan{ + AST: params.GetExpression(), + }, } } return &LokiRequest{ - Query: params.Query(), + Query: params.QueryString(), Limit: params.Limit(), Step: params.Step().Milliseconds(), Interval: params.Interval().Milliseconds(), @@ -54,7 +58,10 @@ func ParamsToLokiRequest(params logql.Params, shards logql.Shards) queryrangebas EndTs: params.End(), Direction: params.Direction(), Path: "/loki/api/v1/query_range", // TODO(owen-d): make this derivable - Shards: shards.Encode(), + Shards: params.Shards(), + Plan: &plan.QueryPlan{ + AST: params.GetExpression(), + }, } } @@ -97,12 +104,12 @@ type instance struct { func (in instance) Downstream(ctx context.Context, queries []logql.DownstreamQuery) ([]logqlmodel.Result, error) { return in.For(ctx, queries, func(qry logql.DownstreamQuery) (logqlmodel.Result, error) { - req := ParamsToLokiRequest(qry.Params, qry.Shards).WithQuery(qry.Expr.String()) + req := ParamsToLokiRequest(qry.Params).WithQuery(qry.Params.GetExpression().String()) sp, ctx := opentracing.StartSpanFromContext(ctx, "DownstreamHandler.instance") defer sp.Finish() logger := spanlogger.FromContext(ctx) defer logger.Finish() - level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Shards), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler)) + level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Params.Shards()), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler)) res, err := in.handler.Do(ctx, req) if err != nil { diff --git a/pkg/querier/queryrange/downstreamer_test.go b/pkg/querier/queryrange/downstreamer_test.go index 552c0c53aa..e453f03d9a 100644 --- a/pkg/querier/queryrange/downstreamer_test.go +++ b/pkg/querier/queryrange/downstreamer_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "strconv" + "strings" "sync" "testing" "time" @@ -223,8 +225,8 @@ func TestInstanceFor(t *testing.T) { } in := mkIn() newParams := func() logql.Params { - return logql.NewLiteralParams( - "", + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Now(), time.Now(), 0, @@ -233,6 +235,8 @@ func TestInstanceFor(t *testing.T) { 1000, nil, ) + require.NoError(t, err) + return params } var queries []logql.DownstreamQuery @@ -280,22 +284,32 @@ func TestInstanceFor(t *testing.T) { context.TODO(), []logql.DownstreamQuery{ { - Params: newParams(), - Shards: logql.Shards{ - {Shard: 0, Of: 2}, + Params: logql.ParamsWithShardsOverride{ + Params: newParams(), + ShardsOverride: logql.Shards{ + {Shard: 0, Of: 2}, + }.Encode(), }, }, { - Params: newParams(), - Shards: logql.Shards{ - {Shard: 1, Of: 2}, + Params: logql.ParamsWithShardsOverride{ + Params: newParams(), + ShardsOverride: logql.Shards{ + {Shard: 1, Of: 2}, + }.Encode(), }, }, }, func(qry logql.DownstreamQuery) (logqlmodel.Result, error) { + // Decode shard + s := strings.Split(qry.Params.Shards()[0], "_") + shard, err := strconv.Atoi(s[0]) + if err != nil { + return logqlmodel.Result{}, err + } return logqlmodel.Result{ Data: promql.Scalar{ - V: float64(qry.Shards[0].Shard), + V: float64(shard), }, }, nil }, @@ -309,8 +323,8 @@ func TestInstanceFor(t *testing.T) { } func TestInstanceDownstream(t *testing.T) { - params := logql.NewLiteralParams( - "", + params, err := logql.NewLiteralParams( + `{foo="bar"}`, time.Now(), time.Now(), 0, @@ -319,8 +333,9 @@ func TestInstanceDownstream(t *testing.T) { 1000, nil, ) + require.NoError(t, err) expr, err := syntax.ParseExpr(`{foo="bar"}`) - require.Nil(t, err) + require.NoError(t, err) expectedResp := func() *LokiResponse { return &LokiResponse{ @@ -340,9 +355,10 @@ func TestInstanceDownstream(t *testing.T) { queries := []logql.DownstreamQuery{ { - Expr: expr, - Params: params, - Shards: logql.Shards{{Shard: 0, Of: 2}}, + Params: logql.ParamsWithShardsOverride{ + Params: logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: expr}, + ShardsOverride: logql.Shards{{Shard: 0, Of: 2}}.Encode(), + }, }, } @@ -353,7 +369,7 @@ func TestInstanceDownstream(t *testing.T) { // for some reason these seemingly can't be checked in their own goroutines, // so we assign them to scoped variables for later comparison. got = req - want = ParamsToLokiRequest(params, queries[0].Shards).WithQuery(expr.String()) + want = ParamsToLokiRequest(queries[0].Params).WithQuery(expr.String()) return expectedResp(), nil }, @@ -484,9 +500,10 @@ func TestDownstreamAccumulatorSimple(t *testing.T) { x = append(x, *s) } // dummy params. Only need to populate direction & limit - params := logql.NewLiteralParams( - "", time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, ) + require.NoError(t, err) acc := newDownstreamAccumulator(params, 1) result := logqlmodel.Result{ @@ -542,9 +559,10 @@ func TestDownstreamAccumulatorMultiMerge(t *testing.T) { } // dummy params. Only need to populate direction & limit - params := logql.NewLiteralParams( - "", time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, + params, err := logql.NewLiteralParams( + `{app="foo"}`, time.Time{}, time.Time{}, 0, 0, direction, uint32(lim), nil, ) + require.NoError(t, err) acc := newDownstreamAccumulator(params, 1) for i := 0; i < nQueries; i++ { diff --git a/pkg/querier/queryrange/marshal.go b/pkg/querier/queryrange/marshal.go index 8b61facf39..0e72b15140 100644 --- a/pkg/querier/queryrange/marshal.go +++ b/pkg/querier/queryrange/marshal.go @@ -6,10 +6,12 @@ import ( "context" "fmt" "io" + "net/http" "time" "github.com/gogo/googleapis/google/rpc" "github.com/gogo/status" + "github.com/grafana/dskit/httpgrpc" "github.com/grafana/dskit/user" "github.com/opentracing/opentracing-go" "github.com/prometheus/prometheus/promql" @@ -19,7 +21,9 @@ import ( "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/logql/sketch" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/util/httpreq" "github.com/grafana/loki/pkg/util/querylimits" @@ -277,12 +281,32 @@ func (Codec) QueryRequestUnwrap(ctx context.Context, req *QueryRequest) (queryra case *QueryRequest_Series: return concrete.Series, ctx, nil case *QueryRequest_Instant: + if concrete.Instant.Plan == nil { + parsed, err := syntax.ParseExpr(concrete.Instant.GetQuery()) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + concrete.Instant.Plan = &plan.QueryPlan{ + AST: parsed, + } + } + return concrete.Instant, ctx, nil case *QueryRequest_Stats: return concrete.Stats, ctx, nil case *QueryRequest_Volume: return concrete.Volume, ctx, nil case *QueryRequest_Streams: + if concrete.Streams.Plan == nil { + parsed, err := syntax.ParseExpr(concrete.Streams.GetQuery()) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + concrete.Streams.Plan = &plan.QueryPlan{ + AST: parsed, + } + } + return concrete.Streams, ctx, nil case *QueryRequest_Labels: return &LabelRequest{ diff --git a/pkg/querier/queryrange/queryrange.pb.go b/pkg/querier/queryrange/queryrange.pb.go index f3955da153..c2cce1dc51 100644 --- a/pkg/querier/queryrange/queryrange.pb.go +++ b/pkg/querier/queryrange/queryrange.pb.go @@ -16,6 +16,7 @@ import ( stats "github.com/grafana/loki/pkg/logqlmodel/stats" _ "github.com/grafana/loki/pkg/push" github_com_grafana_loki_pkg_push "github.com/grafana/loki/pkg/push" + github_com_grafana_loki_pkg_querier_plan "github.com/grafana/loki/pkg/querier/plan" queryrangebase "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" _ "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" github_com_grafana_loki_pkg_querier_queryrange_queryrangebase_definitions "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" @@ -40,15 +41,16 @@ var _ = time.Kitchen const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type LokiRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - Step int64 `protobuf:"varint,3,opt,name=step,proto3" json:"step,omitempty"` - Interval int64 `protobuf:"varint,9,opt,name=interval,proto3" json:"interval,omitempty"` - StartTs time.Time `protobuf:"bytes,4,opt,name=startTs,proto3,stdtime" json:"startTs"` - EndTs time.Time `protobuf:"bytes,5,opt,name=endTs,proto3,stdtime" json:"endTs"` - Direction logproto.Direction `protobuf:"varint,6,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` - Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"` - Shards []string `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + Step int64 `protobuf:"varint,3,opt,name=step,proto3" json:"step,omitempty"` + Interval int64 `protobuf:"varint,9,opt,name=interval,proto3" json:"interval,omitempty"` + StartTs time.Time `protobuf:"bytes,4,opt,name=startTs,proto3,stdtime" json:"startTs"` + EndTs time.Time `protobuf:"bytes,5,opt,name=endTs,proto3,stdtime" json:"endTs"` + Direction logproto.Direction `protobuf:"varint,6,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` + Path string `protobuf:"bytes,7,opt,name=path,proto3" json:"path,omitempty"` + Shards []string `protobuf:"bytes,8,rep,name=shards,proto3" json:"shards"` + Plan *github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,10,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` } func (m *LokiRequest) Reset() { *m = LokiRequest{} } @@ -147,12 +149,13 @@ func (m *LokiRequest) GetShards() []string { } type LokiInstantRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - TimeTs time.Time `protobuf:"bytes,3,opt,name=timeTs,proto3,stdtime" json:"timeTs"` - Direction logproto.Direction `protobuf:"varint,4,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` - Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` - Shards []string `protobuf:"bytes,6,rep,name=shards,proto3" json:"shards"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + Limit uint32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + TimeTs time.Time `protobuf:"bytes,3,opt,name=timeTs,proto3,stdtime" json:"timeTs"` + Direction logproto.Direction `protobuf:"varint,4,opt,name=direction,proto3,enum=logproto.Direction" json:"direction,omitempty"` + Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"` + Shards []string `protobuf:"bytes,6,rep,name=shards,proto3" json:"shards"` + Plan *github_com_grafana_loki_pkg_querier_plan.QueryPlan `protobuf:"bytes,7,opt,name=plan,proto3,customtype=github.com/grafana/loki/pkg/querier/plan.QueryPlan" json:"plan,omitempty"` } func (m *LokiInstantRequest) Reset() { *m = LokiInstantRequest{} } @@ -1123,99 +1126,101 @@ func init() { } var fileDescriptor_51b9d53b40d11902 = []byte{ - // 1458 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x1b, 0x45, - 0x1b, 0xf7, 0xfa, 0x33, 0x9e, 0x34, 0x79, 0xfb, 0x4e, 0xa2, 0x74, 0xdf, 0xb4, 0xef, 0xae, 0x65, - 0x89, 0xd6, 0x20, 0x58, 0x53, 0xa7, 0xf4, 0x13, 0x10, 0x5d, 0xda, 0xca, 0x15, 0x2d, 0x6a, 0x37, - 0x11, 0x07, 0x6e, 0x13, 0x7b, 0x62, 0x2f, 0xf6, 0x7a, 0x37, 0x3b, 0xe3, 0xa8, 0xb9, 0xf1, 0x07, - 0x80, 0xd4, 0xbf, 0x02, 0x21, 0x51, 0x21, 0x71, 0xe5, 0xc8, 0xa9, 0xc7, 0x1c, 0xab, 0x48, 0x18, - 0xea, 0x72, 0x80, 0x9c, 0xfa, 0x27, 0xa0, 0xf9, 0xd8, 0xf5, 0xac, 0xed, 0xb4, 0x76, 0xb9, 0xb4, - 0x12, 0x97, 0x64, 0x3e, 0x9e, 0xdf, 0xec, 0x3c, 0xbf, 0xe7, 0xf7, 0x3c, 0x33, 0x63, 0x70, 0x2e, - 0xe8, 0xb4, 0xaa, 0xbb, 0x7d, 0x1c, 0xba, 0x38, 0xe4, 0xff, 0xf7, 0x43, 0xd4, 0x6b, 0x61, 0xa5, - 0x69, 0x05, 0xa1, 0x4f, 0x7d, 0x08, 0x46, 0x23, 0xeb, 0xb5, 0x96, 0x4b, 0xdb, 0xfd, 0x6d, 0xab, - 0xe1, 0x7b, 0xd5, 0x96, 0xdf, 0xf2, 0xab, 0x2d, 0xdf, 0x6f, 0x75, 0x31, 0x0a, 0x5c, 0x22, 0x9b, - 0xd5, 0x30, 0x68, 0x54, 0x09, 0x45, 0xb4, 0x4f, 0x04, 0x7e, 0x7d, 0x95, 0x19, 0xf2, 0x26, 0x87, - 0xc8, 0x51, 0x53, 0x9a, 0xf3, 0xde, 0x76, 0x7f, 0xa7, 0x4a, 0x5d, 0x0f, 0x13, 0x8a, 0xbc, 0x40, - 0x1a, 0x9c, 0x66, 0xfb, 0xeb, 0xfa, 0x2d, 0x81, 0x8c, 0x1a, 0x72, 0xf2, 0x7f, 0x89, 0x49, 0xd2, - 0xc1, 0xb4, 0xd1, 0x96, 0x53, 0x25, 0x39, 0xb5, 0xdb, 0xf5, 0xfc, 0x26, 0xee, 0xf2, 0xbd, 0x10, - 0xf1, 0x57, 0x5a, 0xac, 0x30, 0x8b, 0xa0, 0x4f, 0xda, 0xfc, 0x8f, 0x1c, 0xfc, 0xf4, 0xa5, 0x74, - 0x6c, 0x23, 0x82, 0xab, 0x4d, 0xbc, 0xe3, 0xf6, 0x5c, 0xea, 0xfa, 0x3d, 0xa2, 0xb6, 0xe5, 0x22, - 0x17, 0x67, 0x5b, 0x64, 0x9c, 0xe2, 0xf2, 0x41, 0x1a, 0x2c, 0xde, 0xf1, 0x3b, 0xae, 0x83, 0x77, - 0xfb, 0x98, 0x50, 0xb8, 0x0a, 0x72, 0xdc, 0x46, 0xd7, 0x4a, 0x5a, 0xa5, 0xe8, 0x88, 0x0e, 0x1b, - 0xed, 0xba, 0x9e, 0x4b, 0xf5, 0x74, 0x49, 0xab, 0x2c, 0x39, 0xa2, 0x03, 0x21, 0xc8, 0x12, 0x8a, - 0x03, 0x3d, 0x53, 0xd2, 0x2a, 0x19, 0x87, 0xb7, 0xe1, 0x3a, 0x58, 0x70, 0x7b, 0x14, 0x87, 0x7b, - 0xa8, 0xab, 0x17, 0xf9, 0x78, 0xdc, 0x87, 0x1f, 0x83, 0x02, 0xa1, 0x28, 0xa4, 0x5b, 0x44, 0xcf, - 0x96, 0xb4, 0xca, 0x62, 0x6d, 0xdd, 0x12, 0xa1, 0xb0, 0xa2, 0x50, 0x58, 0x5b, 0x51, 0x28, 0xec, - 0x85, 0xc7, 0x03, 0x33, 0xf5, 0xf0, 0x37, 0x53, 0x73, 0x22, 0x10, 0xbc, 0x0a, 0x72, 0xb8, 0xd7, - 0xdc, 0x22, 0x7a, 0x6e, 0x0e, 0xb4, 0x80, 0xc0, 0xf3, 0xa0, 0xd8, 0x74, 0x43, 0xdc, 0x60, 0x9c, - 0xe9, 0xf9, 0x92, 0x56, 0x59, 0xae, 0xad, 0x58, 0x71, 0x68, 0x6f, 0x44, 0x53, 0xce, 0xc8, 0x8a, - 0xb9, 0x17, 0x20, 0xda, 0xd6, 0x0b, 0x9c, 0x09, 0xde, 0x86, 0x65, 0x90, 0x27, 0x6d, 0x14, 0x36, - 0x89, 0xbe, 0x50, 0xca, 0x54, 0x8a, 0x36, 0x38, 0x1a, 0x98, 0x72, 0xc4, 0x91, 0xff, 0xcb, 0x7f, - 0x69, 0x00, 0x32, 0x4a, 0x6f, 0xf7, 0x08, 0x45, 0x3d, 0xfa, 0x2a, 0xcc, 0x7e, 0x08, 0xf2, 0x4c, - 0x94, 0x5b, 0x84, 0x73, 0x3b, 0xab, 0xab, 0x12, 0x93, 0xf4, 0x35, 0x3b, 0x97, 0xaf, 0xb9, 0xa9, - 0xbe, 0xe6, 0x8f, 0xf5, 0xf5, 0x87, 0x2c, 0x38, 0x21, 0xe4, 0x43, 0x02, 0xbf, 0x47, 0x30, 0x03, - 0x6d, 0xf2, 0x14, 0x14, 0x6e, 0x4a, 0x10, 0x1f, 0x71, 0xe4, 0x0c, 0xfc, 0x04, 0x64, 0x6f, 0x20, - 0x8a, 0xb8, 0xcb, 0x8b, 0xb5, 0x55, 0x4b, 0x11, 0x25, 0x5b, 0x8b, 0xcd, 0xd9, 0x6b, 0xcc, 0xab, - 0xa3, 0x81, 0xb9, 0xdc, 0x44, 0x14, 0xbd, 0xeb, 0x7b, 0x2e, 0xc5, 0x5e, 0x40, 0xf7, 0x1d, 0x8e, - 0x84, 0x1f, 0x80, 0xe2, 0xcd, 0x30, 0xf4, 0xc3, 0xad, 0xfd, 0x00, 0x73, 0x8a, 0x8a, 0xf6, 0xa9, - 0xa3, 0x81, 0xb9, 0x82, 0xa3, 0x41, 0x05, 0x31, 0xb2, 0x84, 0x6f, 0x83, 0x1c, 0xef, 0x70, 0x52, - 0x8a, 0xf6, 0xca, 0xd1, 0xc0, 0xfc, 0x0f, 0x87, 0x28, 0xe6, 0xc2, 0x22, 0xc9, 0x61, 0x6e, 0x26, - 0x0e, 0xe3, 0x50, 0xe6, 0xd5, 0x50, 0xea, 0xa0, 0xb0, 0x87, 0x43, 0xc2, 0x96, 0x29, 0xf0, 0xf1, - 0xa8, 0x0b, 0xaf, 0x03, 0xc0, 0x88, 0x71, 0x09, 0x75, 0x1b, 0x4c, 0x4f, 0x8c, 0x8c, 0x25, 0x4b, - 0x94, 0x0b, 0x07, 0x93, 0x7e, 0x97, 0xda, 0x50, 0xb2, 0xa0, 0x18, 0x3a, 0x4a, 0x1b, 0x3e, 0xd2, - 0x40, 0xa1, 0x8e, 0x51, 0x13, 0x87, 0x44, 0x2f, 0x96, 0x32, 0x95, 0xc5, 0xda, 0x5b, 0x96, 0x5a, - 0x1b, 0xee, 0x85, 0xbe, 0x87, 0x69, 0x1b, 0xf7, 0x49, 0x14, 0x20, 0x61, 0x6d, 0x77, 0x0e, 0x07, - 0xe6, 0xb6, 0x5a, 0x51, 0x43, 0xb4, 0x83, 0x7a, 0xa8, 0xda, 0xf5, 0x3b, 0x6e, 0x75, 0xee, 0x7a, - 0x74, 0xec, 0x77, 0x8e, 0x06, 0xa6, 0xf6, 0x9e, 0x13, 0x6d, 0xb1, 0xfc, 0xab, 0x06, 0xfe, 0xcb, - 0x22, 0xbc, 0xc9, 0xd6, 0x26, 0x4a, 0x62, 0x78, 0x88, 0x36, 0xda, 0xba, 0xc6, 0x64, 0xe6, 0x88, - 0x8e, 0x5a, 0x2c, 0xd2, 0xff, 0xa8, 0x58, 0x64, 0xe6, 0x2f, 0x16, 0x51, 0x36, 0x64, 0xa7, 0x66, - 0x43, 0xee, 0xd8, 0x6c, 0xf8, 0x26, 0x23, 0x32, 0x3f, 0xf2, 0x6f, 0x8e, 0x9c, 0xb8, 0x15, 0xe7, - 0x44, 0x86, 0xef, 0x36, 0x96, 0x9a, 0x58, 0xeb, 0x76, 0x13, 0xf7, 0xa8, 0xbb, 0xe3, 0xe2, 0xf0, - 0x25, 0x99, 0xa1, 0xc8, 0x2d, 0x93, 0x94, 0x9b, 0xaa, 0x95, 0xec, 0x6b, 0xaf, 0x95, 0xb1, 0xec, - 0xc8, 0xbd, 0x42, 0x76, 0x94, 0x9f, 0xa7, 0xc1, 0x1a, 0x0b, 0xc7, 0x1d, 0xb4, 0x8d, 0xbb, 0x9f, - 0x23, 0x6f, 0xce, 0x90, 0x9c, 0x55, 0x42, 0x52, 0xb4, 0xe1, 0xbf, 0x94, 0xcf, 0x40, 0xf9, 0x77, - 0x1a, 0x58, 0x88, 0x6a, 0x38, 0xb4, 0x00, 0x10, 0x30, 0x5e, 0xa6, 0x05, 0xd1, 0xcb, 0x0c, 0x1c, - 0xc6, 0xa3, 0x8e, 0x62, 0x01, 0xbf, 0x02, 0x79, 0xd1, 0x93, 0x59, 0x70, 0x4a, 0xc9, 0x02, 0x1a, - 0x62, 0xe4, 0x5d, 0x6f, 0xa2, 0x80, 0xe2, 0xd0, 0xbe, 0xc2, 0x76, 0x71, 0x38, 0x30, 0xcf, 0xbd, - 0x88, 0x22, 0x7e, 0xc3, 0x12, 0x38, 0x16, 0x5c, 0xf1, 0x4d, 0x47, 0x7e, 0xa1, 0xfc, 0xad, 0x06, - 0x4e, 0xb2, 0x8d, 0x32, 0x6a, 0x62, 0x55, 0xdc, 0x00, 0x0b, 0xa1, 0x6c, 0xf3, 0xed, 0x2e, 0xd6, - 0xca, 0x56, 0x92, 0xd6, 0x29, 0x54, 0xda, 0xd9, 0xc7, 0x03, 0x53, 0x73, 0x62, 0x24, 0xdc, 0x48, - 0xd0, 0x98, 0x9e, 0x46, 0x23, 0x83, 0xa4, 0x12, 0xc4, 0xfd, 0x9c, 0x06, 0xf0, 0x76, 0xaf, 0x89, - 0x1f, 0x30, 0xf1, 0x8d, 0x74, 0xda, 0x9f, 0xd8, 0xd1, 0x99, 0x11, 0x29, 0x93, 0xf6, 0xf6, 0xb5, - 0xc3, 0x81, 0x79, 0xe9, 0x45, 0xac, 0xbc, 0x00, 0xac, 0xb8, 0xa0, 0x0a, 0x37, 0xfd, 0xfa, 0x9f, - 0x2b, 0x3f, 0xa6, 0xc1, 0xf2, 0x17, 0x7e, 0xb7, 0xef, 0xe1, 0x98, 0x38, 0x6f, 0x82, 0x38, 0x7d, - 0x44, 0x5c, 0xd2, 0xd6, 0xbe, 0x74, 0x38, 0x30, 0x37, 0x66, 0x22, 0x2d, 0x09, 0x7c, 0x73, 0x09, - 0x7b, 0x94, 0x06, 0xab, 0x5b, 0x7e, 0xf0, 0xd9, 0x26, 0x7f, 0xbe, 0x28, 0x75, 0x11, 0x4f, 0xd0, - 0xb6, 0x3a, 0xa2, 0x8d, 0x21, 0xee, 0x22, 0x1a, 0xba, 0x0f, 0xec, 0x8d, 0xc3, 0x81, 0x59, 0x9d, - 0x89, 0xb2, 0x11, 0xe8, 0xcd, 0xa5, 0xeb, 0x97, 0x34, 0x58, 0xbb, 0xdf, 0x47, 0x3d, 0xea, 0x76, - 0xb1, 0xa0, 0x2c, 0x26, 0x6c, 0x7f, 0x82, 0x30, 0x63, 0x44, 0x58, 0x12, 0x23, 0xa9, 0xfb, 0xe8, - 0x70, 0x60, 0x5e, 0x99, 0x89, 0xba, 0x69, 0xf0, 0x37, 0x97, 0xc4, 0x9f, 0xb2, 0x60, 0xe9, 0x3e, - 0x5b, 0x25, 0xe6, 0xee, 0x1d, 0x20, 0x8f, 0x5c, 0xc9, 0x1c, 0x8c, 0xee, 0x68, 0x61, 0xd0, 0xb0, - 0x36, 0xe5, 0x61, 0x2c, 0x2c, 0xe0, 0x65, 0x90, 0x27, 0xfc, 0x26, 0x24, 0x0b, 0xaa, 0x31, 0xfe, - 0x6a, 0x48, 0xde, 0xb9, 0xea, 0x29, 0x47, 0xda, 0xb3, 0xb7, 0x54, 0x97, 0x5d, 0x00, 0xa2, 0x9b, - 0x60, 0x79, 0x1c, 0x39, 0x79, 0x3d, 0x60, 0x68, 0x81, 0x81, 0x17, 0x41, 0x8e, 0x57, 0x6e, 0xf9, - 0x62, 0x4d, 0x7c, 0x76, 0xb2, 0x84, 0xd6, 0x53, 0x8e, 0x30, 0x87, 0x35, 0x90, 0x0d, 0x42, 0xdf, - 0x93, 0xa7, 0xe8, 0x99, 0xf1, 0x6f, 0xaa, 0xc7, 0x4e, 0x3d, 0xe5, 0x70, 0x5b, 0x78, 0x81, 0x5d, - 0x79, 0xd9, 0x79, 0x45, 0xf8, 0x13, 0x82, 0x95, 0xac, 0x31, 0x98, 0x02, 0x89, 0x4c, 0xe1, 0x05, - 0x90, 0xdf, 0xe3, 0x65, 0x89, 0xbf, 0x2f, 0xd8, 0xdd, 0x51, 0x01, 0x25, 0x0b, 0x16, 0xf3, 0x4b, - 0xd8, 0xc2, 0x5b, 0xe0, 0x04, 0xf5, 0x83, 0x4e, 0x54, 0x00, 0xe4, 0xf3, 0xa3, 0xa4, 0x62, 0xa7, - 0x15, 0x88, 0x7a, 0xca, 0x49, 0xe0, 0xe0, 0x3d, 0x70, 0x72, 0x37, 0x21, 0x53, 0x4c, 0xf8, 0xbb, - 0x7f, 0x8c, 0xe7, 0xe9, 0xd9, 0x53, 0x4f, 0x39, 0x13, 0x68, 0x1b, 0x8c, 0x32, 0xaa, 0xfc, 0x47, - 0x06, 0x9c, 0x90, 0x9a, 0x11, 0x6f, 0x85, 0x4b, 0xb1, 0x0c, 0x84, 0x64, 0xfe, 0x7f, 0x9c, 0x0c, - 0xb8, 0xb9, 0xa2, 0x82, 0xf7, 0x63, 0x15, 0x08, 0xfd, 0xac, 0x8d, 0xb2, 0x94, 0xc7, 0x5f, 0x41, - 0xc8, 0xc8, 0x6f, 0x44, 0x91, 0x17, 0xb2, 0x39, 0x3d, 0xfd, 0xdc, 0x8d, 0x50, 0x32, 0xec, 0x57, - 0x41, 0xc1, 0x15, 0xcf, 0xfe, 0x69, 0x82, 0x99, 0xfc, 0x55, 0x80, 0x05, 0x52, 0x02, 0xe0, 0xc6, - 0x28, 0xfc, 0x42, 0x35, 0xa7, 0x26, 0xc3, 0x1f, 0x83, 0xa2, 0xe8, 0x9f, 0x8f, 0xa3, 0x9f, 0x97, - 0x98, 0x89, 0xc3, 0x2a, 0x76, 0x4c, 0x86, 0xbe, 0x0e, 0x16, 0x3c, 0x4c, 0x11, 0xbb, 0xcb, 0xea, - 0x05, 0x5e, 0x37, 0xce, 0x26, 0x43, 0x35, 0xe2, 0xdb, 0xba, 0x2b, 0x0d, 0x6f, 0xf6, 0x68, 0xb8, - 0x2f, 0xaf, 0x2d, 0x31, 0x7a, 0xfd, 0x1a, 0x58, 0x4a, 0x18, 0xc0, 0x93, 0x20, 0xd3, 0xc1, 0xd1, - 0x2f, 0x1c, 0xac, 0xc9, 0x1e, 0x77, 0x7b, 0xa8, 0xdb, 0xc7, 0x9c, 0xf6, 0xa2, 0x23, 0x3a, 0x57, - 0xd3, 0x97, 0x35, 0xbb, 0x08, 0x0a, 0xa1, 0xf8, 0x8a, 0xdd, 0x3c, 0x78, 0x6a, 0xa4, 0x9e, 0x3c, - 0x35, 0x52, 0xcf, 0x9f, 0x1a, 0xda, 0xd7, 0x43, 0x43, 0xfb, 0x7e, 0x68, 0x68, 0x8f, 0x87, 0x86, - 0x76, 0x30, 0x34, 0xb4, 0xdf, 0x87, 0x86, 0xf6, 0xe7, 0xd0, 0x48, 0x3d, 0x1f, 0x1a, 0xda, 0xc3, - 0x67, 0x46, 0xea, 0xe0, 0x99, 0x91, 0x7a, 0xf2, 0xcc, 0x48, 0x7d, 0x69, 0xcd, 0x57, 0xc2, 0xb6, - 0xf3, 0x9c, 0x96, 0x8d, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x66, 0x27, 0xc9, 0x7f, 0x7f, 0x14, - 0x00, 0x00, + // 1498 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0x4b, 0x6f, 0xdb, 0xc6, + 0x1a, 0x15, 0xf5, 0xb4, 0xc6, 0x8f, 0x9b, 0x3b, 0x36, 0x1c, 0x5e, 0x27, 0x97, 0x14, 0x04, 0xdc, + 0x44, 0xb7, 0x68, 0xa9, 0xc6, 0x4e, 0xf3, 0x6c, 0x8b, 0x86, 0x4d, 0x02, 0xa7, 0x4d, 0x8a, 0x84, + 0x36, 0xba, 0xe8, 0x6e, 0x2c, 0x8d, 0x25, 0x56, 0x7c, 0x99, 0x33, 0x32, 0xe2, 0x5d, 0x7f, 0x40, + 0x0b, 0xe4, 0x07, 0x74, 0x5d, 0x14, 0x68, 0x50, 0xa0, 0x8b, 0x6e, 0xba, 0xec, 0x2a, 0xcb, 0x2c, + 0x03, 0x01, 0x65, 0x1b, 0xa5, 0x8b, 0xc2, 0xab, 0xfc, 0x84, 0x62, 0x1e, 0xa4, 0x48, 0x49, 0x49, + 0xe4, 0xb4, 0x8b, 0x04, 0xe8, 0x46, 0x9a, 0x19, 0x7e, 0x87, 0x1c, 0x9e, 0x73, 0xbe, 0x6f, 0x66, + 0x08, 0x4e, 0x07, 0xbd, 0x4e, 0x73, 0xaf, 0x8f, 0x43, 0x1b, 0x87, 0xfc, 0xff, 0x20, 0x44, 0x5e, + 0x07, 0xa7, 0x9a, 0x46, 0x10, 0xfa, 0xd4, 0x87, 0x60, 0x34, 0xb2, 0xb6, 0xde, 0xb1, 0x69, 0xb7, + 0xbf, 0x63, 0xb4, 0x7c, 0xb7, 0xd9, 0xf1, 0x3b, 0x7e, 0xb3, 0xe3, 0xfb, 0x1d, 0x07, 0xa3, 0xc0, + 0x26, 0xb2, 0xd9, 0x0c, 0x83, 0x56, 0x93, 0x50, 0x44, 0xfb, 0x44, 0xe0, 0xd7, 0x56, 0x58, 0x20, + 0x6f, 0x72, 0x88, 0x1c, 0xd5, 0x65, 0x38, 0xef, 0xed, 0xf4, 0x77, 0x9b, 0xd4, 0x76, 0x31, 0xa1, + 0xc8, 0x0d, 0x64, 0xc0, 0x09, 0x36, 0x3f, 0xc7, 0xef, 0x08, 0x64, 0xdc, 0x90, 0x17, 0xff, 0x93, + 0xb9, 0x48, 0x7a, 0x98, 0xb6, 0xba, 0xf2, 0x52, 0x4d, 0x5e, 0xda, 0x73, 0x5c, 0xbf, 0x8d, 0x1d, + 0x3e, 0x17, 0x22, 0x7e, 0x65, 0xc4, 0x32, 0x8b, 0x08, 0xfa, 0xa4, 0xcb, 0x7f, 0xe4, 0xe0, 0x87, + 0x2f, 0xa4, 0x63, 0x07, 0x11, 0xdc, 0x6c, 0xe3, 0x5d, 0xdb, 0xb3, 0xa9, 0xed, 0x7b, 0x24, 0xdd, + 0x96, 0x37, 0x39, 0x37, 0xdb, 0x4d, 0xc6, 0x29, 0xae, 0x7f, 0x5d, 0x00, 0xf3, 0x37, 0xfd, 0x9e, + 0x6d, 0xe1, 0xbd, 0x3e, 0x26, 0x14, 0xae, 0x80, 0x12, 0x8f, 0x51, 0x95, 0x9a, 0xd2, 0xa8, 0x5a, + 0xa2, 0xc3, 0x46, 0x1d, 0xdb, 0xb5, 0xa9, 0x9a, 0xaf, 0x29, 0x8d, 0x45, 0x4b, 0x74, 0x20, 0x04, + 0x45, 0x42, 0x71, 0xa0, 0x16, 0x6a, 0x4a, 0xa3, 0x60, 0xf1, 0x36, 0x5c, 0x03, 0x73, 0xb6, 0x47, + 0x71, 0xb8, 0x8f, 0x1c, 0xb5, 0xca, 0xc7, 0x93, 0x3e, 0x7c, 0x1f, 0x54, 0x08, 0x45, 0x21, 0xdd, + 0x26, 0x6a, 0xb1, 0xa6, 0x34, 0xe6, 0xd7, 0xd7, 0x0c, 0x21, 0x85, 0x11, 0x4b, 0x61, 0x6c, 0xc7, + 0x52, 0x98, 0x73, 0x0f, 0x22, 0x3d, 0x77, 0xef, 0x57, 0x5d, 0xb1, 0x62, 0x10, 0xbc, 0x04, 0x4a, + 0xd8, 0x6b, 0x6f, 0x13, 0xb5, 0x74, 0x04, 0xb4, 0x80, 0xc0, 0x33, 0xa0, 0xda, 0xb6, 0x43, 0xdc, + 0x62, 0x9c, 0xa9, 0xe5, 0x9a, 0xd2, 0x58, 0x5a, 0x5f, 0x36, 0x12, 0x69, 0xaf, 0xc6, 0x97, 0xac, + 0x51, 0x14, 0x7b, 0xbd, 0x00, 0xd1, 0xae, 0x5a, 0xe1, 0x4c, 0xf0, 0x36, 0xac, 0x83, 0x32, 0xe9, + 0xa2, 0xb0, 0x4d, 0xd4, 0xb9, 0x5a, 0xa1, 0x51, 0x35, 0xc1, 0x61, 0xa4, 0xcb, 0x11, 0x4b, 0xfe, + 0xc3, 0x8f, 0x40, 0x31, 0x70, 0x90, 0xa7, 0x82, 0x9a, 0xd2, 0x58, 0x30, 0xcf, 0x0d, 0x22, 0x3d, + 0xe3, 0xdd, 0x10, 0xed, 0x22, 0x0f, 0x35, 0x1d, 0xbf, 0x67, 0x37, 0xd3, 0xa2, 0x31, 0x8c, 0x71, + 0x87, 0xd1, 0x7d, 0xdb, 0x41, 0x9e, 0xc5, 0xef, 0x51, 0xff, 0x31, 0x0f, 0x20, 0x93, 0xe7, 0x86, + 0x47, 0x28, 0xf2, 0xe8, 0xcb, 0xa8, 0xf4, 0x2e, 0x28, 0x33, 0x83, 0x6f, 0x13, 0xae, 0xd3, 0xac, + 0xb4, 0x49, 0x4c, 0x96, 0xb7, 0xe2, 0x91, 0x78, 0x2b, 0x4d, 0xe5, 0xad, 0xfc, 0x42, 0xde, 0x2a, + 0x7f, 0x03, 0x6f, 0xdf, 0x15, 0xc1, 0x82, 0xb0, 0x35, 0x09, 0x7c, 0x8f, 0x60, 0x36, 0x81, 0x2d, + 0x5e, 0x1a, 0x04, 0x65, 0x72, 0x02, 0x7c, 0xc4, 0x92, 0x57, 0xe0, 0x07, 0xa0, 0x78, 0x15, 0x51, + 0xc4, 0xe9, 0x9b, 0x5f, 0x5f, 0x31, 0x52, 0xc9, 0xc2, 0xee, 0xc5, 0xae, 0x99, 0xab, 0x8c, 0xa1, + 0xc3, 0x48, 0x5f, 0x6a, 0x23, 0x8a, 0xde, 0xf4, 0x5d, 0x9b, 0x62, 0x37, 0xa0, 0x07, 0x16, 0x47, + 0xc2, 0x77, 0x40, 0xf5, 0x5a, 0x18, 0xfa, 0xe1, 0xf6, 0x41, 0x80, 0x39, 0xdd, 0x55, 0xf3, 0xf8, + 0x61, 0xa4, 0x2f, 0xe3, 0x78, 0x30, 0x85, 0x18, 0x45, 0xc2, 0xff, 0x83, 0x12, 0xef, 0x70, 0x82, + 0xab, 0xe6, 0xf2, 0x61, 0xa4, 0xff, 0x8b, 0x43, 0x52, 0xe1, 0x22, 0x22, 0xab, 0x47, 0x69, 0x26, + 0x3d, 0x12, 0x5b, 0x94, 0xd3, 0xb6, 0x50, 0x41, 0x65, 0x1f, 0x87, 0x84, 0xdd, 0xa6, 0xc2, 0xc7, + 0xe3, 0x2e, 0xbc, 0x02, 0x00, 0x23, 0xc6, 0x26, 0xd4, 0x6e, 0x31, 0x9f, 0x33, 0x32, 0x16, 0x0d, + 0x51, 0xc6, 0x2c, 0x4c, 0xfa, 0x0e, 0x35, 0xa1, 0x64, 0x21, 0x15, 0x68, 0xa5, 0xda, 0xf0, 0xbe, + 0x02, 0x2a, 0x9b, 0x18, 0xb5, 0x71, 0x48, 0xd4, 0x6a, 0xad, 0xd0, 0x98, 0x5f, 0xff, 0x9f, 0x91, + 0xae, 0x59, 0xb7, 0x43, 0xdf, 0xc5, 0xb4, 0x8b, 0xfb, 0x24, 0x16, 0x48, 0x44, 0x9b, 0xbd, 0x41, + 0xa4, 0xef, 0xcc, 0xa2, 0xfa, 0x4c, 0x75, 0xf2, 0x99, 0xcf, 0x39, 0x8c, 0x74, 0xe5, 0x2d, 0x2b, + 0x9e, 0x62, 0xfd, 0x17, 0x05, 0xfc, 0x9b, 0x29, 0xbc, 0xc5, 0xee, 0x4d, 0x52, 0x49, 0xe6, 0x22, + 0xda, 0xea, 0xaa, 0x0a, 0xb3, 0xac, 0x25, 0x3a, 0xe9, 0x22, 0x96, 0xff, 0x4b, 0x45, 0xac, 0x70, + 0xf4, 0x22, 0x16, 0x67, 0x56, 0x71, 0x6a, 0x66, 0x95, 0x9e, 0x95, 0x59, 0xf5, 0x2f, 0x0b, 0xa2, + 0x8a, 0xc4, 0xef, 0x77, 0x84, 0x9c, 0xb8, 0x9e, 0xe4, 0x44, 0x81, 0xcf, 0x36, 0xb1, 0x9a, 0xb8, + 0xd7, 0x8d, 0x36, 0xf6, 0xa8, 0xbd, 0x6b, 0xe3, 0xf0, 0x05, 0x99, 0x91, 0xb2, 0x5b, 0x21, 0x6b, + 0xb7, 0xb4, 0x57, 0x8a, 0xaf, 0xbc, 0x57, 0xc6, 0xb2, 0xa3, 0xf4, 0x12, 0xd9, 0x51, 0x7f, 0x9a, + 0x07, 0xab, 0x4c, 0x8e, 0x9b, 0x68, 0x07, 0x3b, 0x9f, 0x20, 0xf7, 0x88, 0x92, 0x9c, 0x4a, 0x49, + 0x52, 0x35, 0xe1, 0x3f, 0x94, 0xcf, 0x40, 0xf9, 0x37, 0x0a, 0x98, 0x8b, 0x6b, 0x38, 0x34, 0x00, + 0x10, 0x30, 0x5e, 0xa6, 0x05, 0xd1, 0x4b, 0x0c, 0x1c, 0x26, 0xa3, 0x56, 0x2a, 0x02, 0x7e, 0x0e, + 0xca, 0xa2, 0x27, 0xb3, 0xe0, 0x78, 0x2a, 0x0b, 0x68, 0x88, 0x91, 0x7b, 0xa5, 0x8d, 0x02, 0x8a, + 0x43, 0xf3, 0x22, 0x9b, 0xc5, 0x20, 0xd2, 0x4f, 0x3f, 0x8f, 0x22, 0xbe, 0xf3, 0x13, 0x38, 0x26, + 0xae, 0x78, 0xa6, 0x25, 0x9f, 0x50, 0xff, 0x4a, 0x01, 0xc7, 0xd8, 0x44, 0x19, 0x35, 0x89, 0x2b, + 0xae, 0x82, 0xb9, 0x50, 0xb6, 0xf9, 0x74, 0xe7, 0xd7, 0xeb, 0x46, 0x96, 0xd6, 0x29, 0x54, 0x9a, + 0xc5, 0x07, 0x91, 0xae, 0x58, 0x09, 0x12, 0x6e, 0x64, 0x68, 0xcc, 0x4f, 0xa3, 0x91, 0x41, 0x72, + 0x19, 0xe2, 0x7e, 0xca, 0x03, 0x78, 0xc3, 0x6b, 0xe3, 0xbb, 0xcc, 0x7c, 0x23, 0x9f, 0xf6, 0x27, + 0x66, 0x74, 0x72, 0x44, 0xca, 0x64, 0xbc, 0x79, 0x79, 0x10, 0xe9, 0xe7, 0x9f, 0xc7, 0xca, 0x73, + 0xc0, 0xa9, 0x57, 0x48, 0x1b, 0x37, 0xff, 0xea, 0xaf, 0x2b, 0xdf, 0xe7, 0xc1, 0xd2, 0xa7, 0xbe, + 0xd3, 0x77, 0x71, 0x42, 0x9c, 0x3b, 0x41, 0x9c, 0x3a, 0x22, 0x2e, 0x1b, 0x6b, 0x9e, 0x1f, 0x44, + 0xfa, 0xc6, 0x4c, 0xa4, 0x65, 0x81, 0xaf, 0x2f, 0x61, 0xf7, 0xf3, 0x60, 0x65, 0xdb, 0x0f, 0x3e, + 0xde, 0xe2, 0xc7, 0xaa, 0x54, 0x5d, 0xc4, 0x13, 0xb4, 0xad, 0x8c, 0x68, 0x63, 0x88, 0x5b, 0x88, + 0x86, 0xf6, 0x5d, 0x73, 0x63, 0x10, 0xe9, 0xcd, 0x99, 0x28, 0x1b, 0x81, 0x5e, 0x5f, 0xba, 0x7e, + 0xce, 0x83, 0xd5, 0x3b, 0x7d, 0xe4, 0x51, 0xdb, 0xc1, 0x82, 0xb2, 0x84, 0xb0, 0x83, 0x09, 0xc2, + 0xb4, 0x11, 0x61, 0x59, 0x8c, 0xa4, 0xee, 0xbd, 0x41, 0xa4, 0x5f, 0x9c, 0x89, 0xba, 0x69, 0xf0, + 0xd7, 0x97, 0xc4, 0x1f, 0x8a, 0x60, 0x91, 0x1f, 0x1f, 0x12, 0xee, 0xde, 0x00, 0x72, 0xc9, 0x95, + 0xcc, 0xc1, 0x78, 0x8f, 0x16, 0x06, 0x2d, 0x63, 0x4b, 0x2e, 0xc6, 0x22, 0x02, 0x5e, 0x00, 0x65, + 0xc2, 0x77, 0x42, 0xb2, 0xa0, 0x6a, 0xe3, 0xa7, 0x86, 0xec, 0x9e, 0x6b, 0x33, 0x67, 0xc9, 0x78, + 0x76, 0x2e, 0x73, 0xd8, 0x06, 0x20, 0xde, 0x09, 0xd6, 0xc7, 0x91, 0x93, 0xdb, 0x03, 0x86, 0x16, + 0x18, 0x78, 0x0e, 0x94, 0x78, 0xe5, 0x96, 0x27, 0xe9, 0xcc, 0x63, 0x27, 0x4b, 0xe8, 0x66, 0xce, + 0x12, 0xe1, 0x70, 0x1d, 0x14, 0x83, 0xd0, 0x77, 0xe5, 0x2a, 0x7a, 0x72, 0xfc, 0x99, 0xe9, 0x65, + 0x67, 0x33, 0x67, 0xf1, 0x58, 0x78, 0x96, 0x6d, 0x79, 0xd9, 0x7a, 0x45, 0xf8, 0x11, 0x82, 0x95, + 0xac, 0x31, 0x58, 0x0a, 0x12, 0x87, 0xc2, 0xb3, 0xa0, 0xbc, 0xcf, 0xcb, 0x12, 0x3f, 0x5f, 0xb0, + 0xbd, 0x63, 0x0a, 0x94, 0x2d, 0x58, 0xec, 0xbd, 0x44, 0x2c, 0xbc, 0x0e, 0x16, 0xa8, 0x1f, 0xf4, + 0xe2, 0x02, 0x20, 0x8f, 0x1f, 0xb5, 0x34, 0x76, 0x5a, 0x81, 0xd8, 0xcc, 0x59, 0x19, 0x1c, 0xbc, + 0x0d, 0x8e, 0xed, 0x65, 0x6c, 0x8a, 0x09, 0xff, 0x1e, 0x31, 0xc6, 0xf3, 0xf4, 0xec, 0xd9, 0xcc, + 0x59, 0x13, 0x68, 0x13, 0x8c, 0x32, 0xaa, 0xfe, 0x7b, 0x01, 0x2c, 0x48, 0xcf, 0x88, 0xb3, 0xc2, + 0xf9, 0xc4, 0x06, 0xc2, 0x32, 0xff, 0x7d, 0x96, 0x0d, 0x78, 0x78, 0xca, 0x05, 0x6f, 0x27, 0x2e, + 0x10, 0xfe, 0x59, 0x1d, 0x65, 0x29, 0xd7, 0x3f, 0x85, 0x90, 0xca, 0x6f, 0xc4, 0xca, 0x0b, 0xdb, + 0x9c, 0x98, 0xbe, 0xee, 0xc6, 0x28, 0x29, 0xfb, 0x25, 0x50, 0xb1, 0xc5, 0x27, 0x84, 0x69, 0x86, + 0x99, 0xfc, 0xc2, 0xc0, 0x84, 0x94, 0x00, 0xb8, 0x31, 0x92, 0x5f, 0xb8, 0xe6, 0xf8, 0xa4, 0xfc, + 0x09, 0x28, 0x56, 0xff, 0x4c, 0xa2, 0x7e, 0x59, 0x62, 0x26, 0x16, 0xab, 0xe4, 0xc5, 0xa4, 0xf4, + 0x9b, 0x60, 0xce, 0xc5, 0x14, 0xb1, 0xbd, 0xac, 0x5a, 0xe1, 0x75, 0xe3, 0x54, 0x56, 0xaa, 0x11, + 0xdf, 0xc6, 0x2d, 0x19, 0x78, 0xcd, 0xa3, 0xe1, 0x81, 0xdc, 0xb6, 0x24, 0xe8, 0xb5, 0xcb, 0x60, + 0x31, 0x13, 0x00, 0x8f, 0x81, 0x42, 0x0f, 0xc7, 0x5f, 0x4b, 0x58, 0x93, 0x1d, 0xee, 0xf6, 0x91, + 0xd3, 0xc7, 0x9c, 0xf6, 0xaa, 0x25, 0x3a, 0x97, 0xf2, 0x17, 0x14, 0xb3, 0x0a, 0x2a, 0xa1, 0x78, + 0x8a, 0xd9, 0x7e, 0xf8, 0x58, 0xcb, 0x3d, 0x7a, 0xac, 0xe5, 0x9e, 0x3e, 0xd6, 0x94, 0x2f, 0x86, + 0x9a, 0xf2, 0xed, 0x50, 0x53, 0x1e, 0x0c, 0x35, 0xe5, 0xe1, 0x50, 0x53, 0x7e, 0x1b, 0x6a, 0xca, + 0x1f, 0x43, 0x2d, 0xf7, 0x74, 0xa8, 0x29, 0xf7, 0x9e, 0x68, 0xb9, 0x87, 0x4f, 0xb4, 0xdc, 0xa3, + 0x27, 0x5a, 0xee, 0x33, 0xe3, 0x68, 0x25, 0x6c, 0xa7, 0xcc, 0x69, 0xd9, 0xf8, 0x33, 0x00, 0x00, + 0xff, 0xff, 0x33, 0xb4, 0xee, 0x07, 0x17, 0x15, 0x00, 0x00, } func (this *LokiRequest) Equal(that interface{}) bool { @@ -1269,6 +1274,13 @@ func (this *LokiRequest) Equal(that interface{}) bool { return false } } + if that1.Plan == nil { + if this.Plan != nil { + return false + } + } else if !this.Plan.Equal(*that1.Plan) { + return false + } return true } func (this *LokiInstantRequest) Equal(that interface{}) bool { @@ -1313,6 +1325,13 @@ func (this *LokiInstantRequest) Equal(that interface{}) bool { return false } } + if that1.Plan == nil { + if this.Plan != nil { + return false + } + } else if !this.Plan.Equal(*that1.Plan) { + return false + } return true } func (this *LokiResponse) Equal(that interface{}) bool { @@ -2120,7 +2139,7 @@ func (this *LokiRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 13) + s := make([]string, 0, 14) s = append(s, "&queryrange.LokiRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2131,6 +2150,7 @@ func (this *LokiRequest) GoString() string { s = append(s, "Direction: "+fmt.Sprintf("%#v", this.Direction)+",\n") s = append(s, "Path: "+fmt.Sprintf("%#v", this.Path)+",\n") s = append(s, "Shards: "+fmt.Sprintf("%#v", this.Shards)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2138,7 +2158,7 @@ func (this *LokiInstantRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&queryrange.LokiInstantRequest{") s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") @@ -2146,6 +2166,7 @@ func (this *LokiInstantRequest) GoString() string { s = append(s, "Direction: "+fmt.Sprintf("%#v", this.Direction)+",\n") s = append(s, "Path: "+fmt.Sprintf("%#v", this.Path)+",\n") s = append(s, "Shards: "+fmt.Sprintf("%#v", this.Shards)+",\n") + s = append(s, "Plan: "+fmt.Sprintf("%#v", this.Plan)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2463,6 +2484,18 @@ func (m *LokiRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Plan != nil { + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } if m.Interval != 0 { i = encodeVarintQueryrange(dAtA, i, uint64(m.Interval)) i-- @@ -2545,6 +2578,18 @@ func (m *LokiInstantRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Plan != nil { + { + size := m.Plan.Size() + i -= size + if _, err := m.Plan.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintQueryrange(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if len(m.Shards) > 0 { for iNdEx := len(m.Shards) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Shards[iNdEx]) @@ -3594,6 +3639,10 @@ func (m *LokiRequest) Size() (n int) { if m.Interval != 0 { n += 1 + sovQueryrange(uint64(m.Interval)) } + if m.Plan != nil { + l = m.Plan.Size() + n += 1 + l + sovQueryrange(uint64(l)) + } return n } @@ -3625,6 +3674,10 @@ func (m *LokiInstantRequest) Size() (n int) { n += 1 + l + sovQueryrange(uint64(l)) } } + if m.Plan != nil { + l = m.Plan.Size() + n += 1 + l + sovQueryrange(uint64(l)) + } return n } @@ -4092,6 +4145,7 @@ func (this *LokiRequest) String() string { `Path:` + fmt.Sprintf("%v", this.Path) + `,`, `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, `Interval:` + fmt.Sprintf("%v", this.Interval) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -4107,6 +4161,7 @@ func (this *LokiInstantRequest) String() string { `Direction:` + fmt.Sprintf("%v", this.Direction) + `,`, `Path:` + fmt.Sprintf("%v", this.Path) + `,`, `Shards:` + fmt.Sprintf("%v", this.Shards) + `,`, + `Plan:` + fmt.Sprintf("%v", this.Plan) + `,`, `}`, }, "") return s @@ -4689,6 +4744,41 @@ func (m *LokiRequest) Unmarshal(dAtA []byte) error { break } } + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var v github_com_grafana_loki_pkg_querier_plan.QueryPlan + m.Plan = &v + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) @@ -4909,6 +4999,41 @@ func (m *LokiInstantRequest) Unmarshal(dAtA []byte) error { } m.Shards = append(m.Shards, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Plan", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var v github_com_grafana_loki_pkg_querier_plan.QueryPlan + m.Plan = &v + if err := m.Plan.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) diff --git a/pkg/querier/queryrange/queryrange.proto b/pkg/querier/queryrange/queryrange.proto index d5e89eeee4..8eb43e34ca 100644 --- a/pkg/querier/queryrange/queryrange.proto +++ b/pkg/querier/queryrange/queryrange.proto @@ -33,6 +33,7 @@ message LokiRequest { logproto.Direction direction = 6; string path = 7; repeated string shards = 8 [(gogoproto.jsontag) = "shards"]; + bytes plan = 10 [(gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan"]; } message LokiInstantRequest { @@ -45,6 +46,7 @@ message LokiInstantRequest { logproto.Direction direction = 4; string path = 5; repeated string shards = 6 [(gogoproto.jsontag) = "shards"]; + bytes plan = 7 [(gogoproto.customtype) = "github.com/grafana/loki/pkg/querier/plan.QueryPlan"]; } message LokiResponse { diff --git a/pkg/querier/queryrange/querysharding.go b/pkg/querier/queryrange/querysharding.go index 26ec924ce5..1df7bb4616 100644 --- a/pkg/querier/queryrange/querysharding.go +++ b/pkg/querier/queryrange/querysharding.go @@ -185,7 +185,12 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que mapper := logql.NewShardMapper(resolver, ast.metrics) - noop, bytesPerShard, parsed, err := mapper.Parse(r.GetQuery()) + params, err := ParamsFromRequest(r) + if err != nil { + return nil, err + } + + noop, bytesPerShard, parsed, err := mapper.Parse(params.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "failed mapping AST", "err", err.Error(), "query", r.GetQuery()) return nil, err @@ -203,11 +208,6 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que return ast.next.Do(ctx, r) } - params, err := ParamsFromRequest(r) - if err != nil { - return nil, err - } - var path string switch r := r.(type) { case *LokiRequest: @@ -217,7 +217,7 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que default: return nil, fmt.Errorf("expected *LokiRequest or *LokiInstantRequest, got (%T)", r) } - query := ast.ng.Query(ctx, params, parsed) + query := ast.ng.Query(ctx, logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: parsed}) res, err := query.Exec(ctx) if err != nil { diff --git a/pkg/querier/queryrange/querysharding_test.go b/pkg/querier/queryrange/querysharding_test.go index 8c77afe041..d3d3ce807a 100644 --- a/pkg/querier/queryrange/querysharding_test.go +++ b/pkg/querier/queryrange/querysharding_test.go @@ -19,7 +19,9 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase/definitions" "github.com/grafana/loki/pkg/storage/config" @@ -172,7 +174,12 @@ func Test_astMapper(t *testing.T) { 0, ) - resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(`{food="bar"}`)) + req := defaultReq() + req.Query = `{foo="bar"}` + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(req.Query), + } + resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) require.Nil(t, err) require.Equal(t, []*definitions.PrometheusResponseHeader{ @@ -311,7 +318,12 @@ func Test_astMapper_QuerySizeLimits(t *testing.T) { 0, ) - _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(tc.query)) + req := defaultReq() + req.Query = tc.query + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(tc.query), + } + _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) if err != nil { require.ErrorContains(t, err, tc.err) } @@ -344,7 +356,13 @@ func Test_ShardingByPass(t *testing.T) { 0, ) - _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), defaultReq().WithQuery(`1+1`)) + req := defaultReq() + req.Query = `1+1` + req.Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(req.Query), + } + + _, err := mware.Do(user.InjectOrgID(context.Background(), "1"), req) require.Nil(t, err) require.Equal(t, called, 1) } @@ -437,6 +455,9 @@ func Test_InstantSharding(t *testing.T) { Query: `rate({app="foo"}[1m])`, TimeTs: util.TimeFromMillis(10), Path: "/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`rate({app="foo"}[1m])`), + }, }) require.NoError(t, err) require.Equal(t, 3, called, "expected 3 calls but got {}", called) @@ -703,6 +724,13 @@ func TestShardingAcrossConfigs_ASTMapper(t *testing.T) { 0, ) + // currently all the tests call `defaultReq()` which creates an instance of the type LokiRequest + // if in the future that isn't true, we need another way to access the Plan field of an arbitrary query type + // or we should set the Plan in calls to `GetExpression` if the Plan is nil by calling `ParseExpr` or similar + tc.req.(*LokiRequest).Plan = &plan.QueryPlan{ + AST: syntax.MustParseExpr(tc.req.GetQuery()), + } + resp, err := mware.Do(user.InjectOrgID(context.Background(), "1"), tc.req) require.Nil(t, err) @@ -830,12 +858,16 @@ func Test_ASTMapper_MaxLookBackPeriod(t *testing.T) { 0, ) + q := `{cluster="dev-us-central-0"}` lokiReq := &LokiInstantRequest{ - Query: `{cluster="dev-us-central-0"}`, + Query: q, Limit: 1000, TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(q), + }, } ctx := user.InjectOrgID(context.Background(), "foo") diff --git a/pkg/querier/queryrange/roundtrip_test.go b/pkg/querier/queryrange/roundtrip_test.go index 4c19a1ffc1..b91e088041 100644 --- a/pkg/querier/queryrange/roundtrip_test.go +++ b/pkg/querier/queryrange/roundtrip_test.go @@ -23,8 +23,10 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" + "github.com/grafana/loki/pkg/logql/syntax" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" + "github.com/grafana/loki/pkg/querier/plan" base "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/storage/chunk/cache" "github.com/grafana/loki/pkg/storage/config" @@ -332,12 +334,16 @@ func TestInstantQueryTripperware(t *testing.T) { } require.NoError(t, err) + q := `sum by (job) (bytes_rate({cluster="dev-us-central-0"}[15m]))` lreq := &LokiInstantRequest{ - Query: `sum by (job) (bytes_rate({cluster="dev-us-central-0"}[15m]))`, + Query: q, Limit: 1000, TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(q), + }, } ctx := user.InjectOrgID(context.Background(), "1") @@ -1101,6 +1107,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[2h]))`), + }, }, expectedSplitStats: 2, // [2h] interval split by 1h configured split interval expectedShardStats: 8, // 2 time splits * 4 row shards @@ -1113,6 +1122,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { TimeTs: testTime, Direction: logproto.FORWARD, Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 0, // [1h] interval not split expectedShardStats: 4, // 4 row shards @@ -1127,6 +1139,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { EndTs: testTime, Direction: logproto.FORWARD, Path: "/query_range", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 3, // 2 hour range interval split based on the base hour + the remainder expectedShardStats: 12, // 3 time splits * 4 row shards @@ -1141,6 +1156,9 @@ func TestMetricsTripperware_SplitShardStats(t *testing.T) { EndTs: testTime, Direction: logproto.FORWARD, Path: "/query_range", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (app) (rate({app="foo"} |= "foo"[1h]))`), + }, }, expectedSplitStats: 0, // 1 minute range interval not split expectedShardStats: 4, // 4 row shards diff --git a/pkg/querier/queryrange/split_by_interval.go b/pkg/querier/queryrange/split_by_interval.go index da8326a678..b4245375bc 100644 --- a/pkg/querier/queryrange/split_by_interval.go +++ b/pkg/querier/queryrange/split_by_interval.go @@ -383,6 +383,7 @@ func splitMetricByTime(r queryrangebase.Request, interval time.Duration) ([]quer Path: lokiReq.Path, StartTs: start, EndTs: end, + Plan: lokiReq.Plan, }) }) @@ -403,6 +404,7 @@ func splitMetricByTime(r queryrangebase.Request, interval time.Duration) ([]quer Path: lokiReq.Path, StartTs: start, EndTs: end, + Plan: lokiReq.Plan, }) } diff --git a/pkg/querier/queryrange/split_by_range.go b/pkg/querier/queryrange/split_by_range.go index e3640761d5..6845846d4d 100644 --- a/pkg/querier/queryrange/split_by_range.go +++ b/pkg/querier/queryrange/split_by_range.go @@ -47,6 +47,11 @@ func NewSplitByRangeMiddleware(logger log.Logger, engineOpts logql.EngineOpts, l func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) (queryrangebase.Response, error) { logger := util_log.WithContext(ctx, s.logger) + params, err := ParamsFromRequest(request) + if err != nil { + return nil, err + } + tenants, err := tenant.TenantIDs(ctx) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) @@ -64,7 +69,7 @@ func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) ( return nil, err } - noop, parsed, err := mapper.Parse(request.GetQuery()) + noop, parsed, err := mapper.Parse(params.GetExpression()) if err != nil { level.Warn(logger).Log("msg", "failed mapping AST", "err", err.Error(), "query", request.GetQuery()) return nil, err @@ -80,16 +85,11 @@ func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) ( queryStatsCtx := stats.FromContext(ctx) queryStatsCtx.AddSplitQueries(int64(mapperStats.GetSplitQueries())) - params, err := ParamsFromRequest(request) - if err != nil { - return nil, err - } - if _, ok := request.(*LokiInstantRequest); !ok { - return nil, fmt.Errorf("expected *LokiInstantRequest") + return nil, fmt.Errorf("expected *LokiInstantRequest, got %T", request) } - query := s.ng.Query(ctx, params, parsed) + query := s.ng.Query(ctx, logql.ParamsWithExpressionOverride{Params: params, ExpressionOverride: parsed}) res, err := query.Exec(ctx) if err != nil { diff --git a/pkg/querier/queryrange/split_by_range_test.go b/pkg/querier/queryrange/split_by_range_test.go index c3b4587a1d..ef25e3f910 100644 --- a/pkg/querier/queryrange/split_by_range_test.go +++ b/pkg/querier/queryrange/split_by_range_test.go @@ -6,13 +6,14 @@ import ( "testing" "time" - "github.com/grafana/loki/pkg/loghttp" - "github.com/go-kit/log" "github.com/grafana/dskit/user" "github.com/stretchr/testify/require" + "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/querier/plan" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" ) @@ -37,6 +38,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(bytes_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(bytes_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(bytes_over_time({app="foo"}[1m]))`, 1), @@ -50,6 +54,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (bytes_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (bytes_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(bytes_over_time({app="foo"}[1m]))`, 10), @@ -63,6 +70,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(count_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(count_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(count_over_time({app="foo"}[1m]))`, 1), @@ -76,6 +86,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (count_over_time({app="foo"}[3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (count_over_time({app="foo"}[3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(count_over_time({app="foo"}[1m]))`, 0), @@ -89,6 +102,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum(sum_over_time({app="foo"} | unwrap bar [3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum(sum_over_time({app="foo"} | unwrap bar [3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum(sum_over_time({app="foo"} | unwrap bar[1m]))`, 1), @@ -102,6 +118,9 @@ func Test_RangeVectorSplit(t *testing.T) { Query: `sum by (bar) (sum_over_time({app="foo"} | unwrap bar [3m]))`, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(`sum by (bar) (sum_over_time({app="foo"} | unwrap bar [3m]))`), + }, }, subQueries: []queryrangebase.RequestResponse{ subQueryRequestResponse(`sum by (bar)(sum_over_time({app="foo"} | unwrap bar[1m]))`, 1), @@ -140,6 +159,9 @@ func subQueryRequestResponse(expectedSubQuery string, sampleValue float64) query Query: expectedSubQuery, TimeTs: time.Unix(1, 0), Path: "/loki/api/v1/query", + Plan: &plan.QueryPlan{ + AST: syntax.MustParseExpr(expectedSubQuery), + }, }, Response: &LokiPromResponse{ Response: &queryrangebase.PrometheusResponse{ diff --git a/pkg/querier/queryrange/stats.go b/pkg/querier/queryrange/stats.go index 0233d886c9..71f93959c3 100644 --- a/pkg/querier/queryrange/stats.go +++ b/pkg/querier/queryrange/stats.go @@ -53,13 +53,13 @@ func recordQueryMetrics(data *queryData) { case queryTypeLog, queryTypeMetric: logql.RecordRangeAndInstantQueryMetrics(data.ctx, logger, data.params, data.status, *data.statistics, data.result) case queryTypeLabel: - logql.RecordLabelQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.label, data.params.Query(), data.status, *data.statistics) + logql.RecordLabelQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.label, data.params.QueryString(), data.status, *data.statistics) case queryTypeSeries: logql.RecordSeriesQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.match, data.status, []string{}, *data.statistics) case queryTypeStats: - logql.RecordStatsQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.Query(), data.status, *data.statistics) + logql.RecordStatsQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.QueryString(), data.status, *data.statistics) case queryTypeVolume: - logql.RecordVolumeQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.Query(), data.params.Limit(), data.params.Step(), data.status, *data.statistics) + logql.RecordVolumeQueryMetrics(data.ctx, logger, data.params.Start(), data.params.End(), data.params.QueryString(), data.params.Limit(), data.params.Step(), data.status, *data.statistics) default: level.Error(logger).Log("msg", "failed to record query metrics", "err", fmt.Errorf("expected one of the *LokiRequest, *LokiInstantRequest, *LokiSeriesRequest, *LokiLabelNamesRequest, got %s", data.queryType)) } diff --git a/pkg/querier/queryrange/stats_test.go b/pkg/querier/queryrange/stats_test.go index 54c9004d88..28f8d12de7 100644 --- a/pkg/querier/queryrange/stats_test.go +++ b/pkg/querier/queryrange/stats_test.go @@ -30,7 +30,7 @@ func TestStatsCollectorMiddleware(t *testing.T) { Query: "foo", StartTs: now, }) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, true, data.recorded) require.Equal(t, now, data.params.Start()) require.Nil(t, data.statistics) @@ -60,7 +60,7 @@ func TestStatsCollectorMiddleware(t *testing.T) { Query: "foo", StartTs: now, }) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, true, data.recorded) require.Equal(t, now, data.params.Start()) require.Equal(t, int32(10), data.statistics.Ingester.TotalReached) @@ -108,7 +108,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusOK), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, stats.Result{}, *data.statistics) @@ -129,7 +129,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) @@ -151,7 +151,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, logproto.BACKWARD, data.params.Direction()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) @@ -173,7 +173,7 @@ func Test_StatsHTTP(t *testing.T) { }), func(t *testing.T, data *queryData) { require.Equal(t, fmt.Sprintf("%d", http.StatusTeapot), data.status) - require.Equal(t, "foo", data.params.Query()) + require.Equal(t, "foo", data.params.QueryString()) require.Equal(t, uint32(100), data.params.Limit()) require.Equal(t, statsResult, *data.statistics) require.Equal(t, streams, data.result) diff --git a/pkg/querier/worker/util_test.go b/pkg/querier/worker/util_test.go index a0213e3bb7..25dd8127a0 100644 --- a/pkg/querier/worker/util_test.go +++ b/pkg/querier/worker/util_test.go @@ -61,7 +61,7 @@ func TestHandleQueryRequest(t *testing.T) { } { t.Run(name, func(t *testing.T) { ctx := user.InjectOrgID(context.Background(), "1") - request, err := queryrange.DefaultCodec.QueryRequestWrap(ctx, &queryrange.LokiRequest{}) + request, err := queryrange.DefaultCodec.QueryRequestWrap(ctx, &queryrange.LokiRequest{Query: `{app="foo"}`}) require.NoError(t, err) mockHandler := HandlerFunc(func(context.Context, queryrangebase.Request) (queryrangebase.Response, error) { diff --git a/pkg/ruler/evaluator_local.go b/pkg/ruler/evaluator_local.go index fed0f2f02e..91efd5a14d 100644 --- a/pkg/ruler/evaluator_local.go +++ b/pkg/ruler/evaluator_local.go @@ -28,7 +28,7 @@ func NewLocalEvaluator(engine *logql.Engine, logger log.Logger) (*LocalEvaluator } func (l *LocalEvaluator) Eval(ctx context.Context, qs string, now time.Time) (*logqlmodel.Result, error) { - params := logql.NewLiteralParams( + params, err := logql.NewLiteralParams( qs, now, now, @@ -38,6 +38,9 @@ func (l *LocalEvaluator) Eval(ctx context.Context, qs string, now time.Time) (*l 0, nil, ) + if err != nil { + return nil, err + } q := l.engine.Query(params) res, err := q.Exec(ctx)