From 9fcc42dc48a9819304c7ef3d760cfa3b515e4ff4 Mon Sep 17 00:00:00 2001 From: Karsten Jeschkies Date: Wed, 18 Oct 2023 16:16:30 +0200 Subject: [PATCH] Support protobuf `QueryRequest` in querier. (#10858) **What this PR does / why we need it**: This is one step to support pure protobuf encoding without `httpgrpc`. It is only on the scheduler and is fully backwards compatible. **Which issue(s) this PR fixes**: This is a sub change of https://github.com/grafana/loki/pull/10688 **Checklist** - [ ] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - [ ] Documentation added - [x] Tests updated - [ ] `CHANGELOG.md` updated - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](https://github.com/grafana/loki/commit/d10549e3ece02120974929894ee333d07755d213) --------- Signed-off-by: Kaviraj Co-authored-by: Kaviraj Co-authored-by: Danny Kopping --- go.mod | 2 +- integration/client/client.go | 36 +- integration/loki_micro_services_test.go | 6 + pkg/loghttp/params.go | 24 +- pkg/loghttp/query.go | 61 +++ pkg/loghttp/tail.go | 7 + pkg/logproto/compat.go | 4 + pkg/loki/loki.go | 2 + pkg/loki/modules.go | 100 ++-- pkg/lokifrontend/frontend/v1/frontend_test.go | 50 +- pkg/querier/handler.go | 105 +++++ pkg/querier/http.go | 275 ++--------- pkg/querier/http_test.go | 375 +-------------- pkg/querier/queryrange/codec.go | 257 +++++++++-- pkg/querier/queryrange/codec_test.go | 40 +- pkg/querier/queryrange/limits_test.go | 11 +- pkg/querier/queryrange/marshal.go | 234 +++------- pkg/querier/queryrange/prometheus.go | 38 +- pkg/querier/queryrange/queryrange.pb.go | 162 +++---- pkg/querier/queryrange/queryrange.proto | 2 + pkg/querier/queryrange/querysharding.go | 2 +- pkg/querier/queryrange/roundtrip_test.go | 8 +- pkg/querier/queryrange/serialize.go | 76 ++++ pkg/querier/queryrange/serialize_test.go | 132 ++++++ pkg/querier/queryrange/split_by_range.go | 2 +- pkg/querier/queryrange/stats.go | 2 +- pkg/querier/queryrange/stats_test.go | 8 +- pkg/querier/queryrange/views_test.go | 5 +- pkg/querier/worker/frontend_processor.go | 36 +- pkg/querier/worker/frontend_processor_test.go | 3 +- pkg/querier/worker/scheduler_processor.go | 18 +- .../worker/scheduler_processor_test.go | 23 +- pkg/querier/worker/util.go | 45 ++ pkg/querier/worker/worker.go | 17 +- pkg/querier/worker_service.go | 101 +--- pkg/querier/worker_service_test.go | 430 +++++++++--------- .../chunk/client/congestion/congestion.go | 18 +- .../client/congestion/controller_test.go | 4 +- pkg/util/marshal/marshal.go | 28 +- pkg/util/marshal/marshal_test.go | 11 +- pkg/util/marshal/query.go | 13 +- 41 files changed, 1415 insertions(+), 1358 deletions(-) create mode 100644 pkg/querier/handler.go create mode 100644 pkg/querier/queryrange/serialize.go create mode 100644 pkg/querier/queryrange/serialize_test.go diff --git a/go.mod b/go.mod index 0618c3c008..f64f68f05f 100644 --- a/go.mod +++ b/go.mod @@ -122,6 +122,7 @@ require ( github.com/d4l3k/messagediff v1.2.1 github.com/efficientgo/core v1.0.0-rc.2 github.com/fsnotify/fsnotify v1.6.0 + github.com/gogo/googleapis v1.4.0 github.com/grafana/loki/pkg/push v0.0.0-20231017172654-cfc4f0e84adc github.com/heroku/x v0.0.61 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b @@ -218,7 +219,6 @@ require ( github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/go-zookeeper/zk v1.0.3 // indirect github.com/gofrs/flock v0.8.1 // indirect - github.com/gogo/googleapis v1.4.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.2 // indirect diff --git a/integration/client/client.go b/integration/client/client.go index a596a2d4fb..8445f271e2 100644 --- a/integration/client/client.go +++ b/integration/client/client.go @@ -511,7 +511,7 @@ func (c *Client) LabelNames(ctx context.Context) ([]string, error) { return values.Data, nil } -// LabelValues return a LabelValues query +// LabelValues return a LabelValues query result func (c *Client) LabelValues(ctx context.Context, labelName string) ([]string, error) { ctx, cancelFunc := context.WithTimeout(ctx, requestTimeout) defer cancelFunc() @@ -543,6 +543,40 @@ func (c *Client) LabelValues(ctx context.Context, labelName string) ([]string, e return values.Data, nil } +// Series return a series query result +func (c *Client) Series(ctx context.Context, matcher string) ([]map[string]string, error) { + ctx, cancelFunc := context.WithTimeout(ctx, requestTimeout) + defer cancelFunc() + + v := url.Values{} + v.Set("match[]", matcher) + + u, err := url.Parse(c.baseURL) + if err != nil { + panic(err) + } + u.Path = "/loki/api/v1/series" + u.RawQuery = v.Encode() + + buf, statusCode, err := c.run(ctx, u.String()) + if err != nil { + return nil, err + } + + if statusCode/100 != 2 { + return nil, fmt.Errorf("request failed with status code %d: %w", statusCode, errors.New(string(buf))) + } + + var values struct { + Data []map[string]string `json:"data"` + } + if err := json.Unmarshal(buf, &values); err != nil { + return nil, err + } + + return values.Data, nil +} + func (c *Client) request(ctx context.Context, method string, url string) (*http.Request, error) { ctx = user.InjectOrgID(ctx, c.instanceID) req, err := http.NewRequestWithContext(ctx, method, url, nil) diff --git a/integration/loki_micro_services_test.go b/integration/loki_micro_services_test.go index 6764870167..db255adabe 100644 --- a/integration/loki_micro_services_test.go +++ b/integration/loki_micro_services_test.go @@ -129,6 +129,12 @@ func TestMicroServicesIngestQuery(t *testing.T) { assert.ElementsMatch(t, []string{"fake"}, resp) }) + t.Run("series", func(t *testing.T) { + resp, err := cliQueryFrontend.Series(context.Background(), `{job="fake"}`) + require.NoError(t, err) + assert.ElementsMatch(t, []map[string]string{{"job": "fake"}}, resp) + }) + t.Run("per-request-limits", func(t *testing.T) { queryLimitsPolicy := client.InjectHeadersOption(map[string][]string{querylimits.HTTPHeaderQueryLimitsKey: {`{"maxQueryLength": "1m"}`}}) cliQueryFrontendLimited := client.New(tenantID, "", tQueryFrontend.HTTPURL(), queryLimitsPolicy) diff --git a/pkg/loghttp/params.go b/pkg/loghttp/params.go index 6a96fa97ff..df97a5c2e3 100644 --- a/pkg/loghttp/params.go +++ b/pkg/loghttp/params.go @@ -10,13 +10,16 @@ import ( "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" ) const ( defaultQueryLimit = 100 defaultSince = 1 * time.Hour + defaultDirection = logproto.BACKWARD ) func limit(r *http.Request) (uint32, error) { @@ -39,7 +42,7 @@ func ts(r *http.Request) (time.Time, error) { } func direction(r *http.Request) (logproto.Direction, error) { - return parseDirection(r.Form.Get("direction"), logproto.BACKWARD) + return parseDirection(r.Form.Get("direction"), defaultDirection) } func shards(r *http.Request) []string { @@ -178,3 +181,22 @@ func parseSecondsOrDuration(value string) (time.Duration, error) { } return 0, errors.Errorf("cannot parse %q to a valid duration", value) } + +// parseRegexQuery parses regex and query querystring from httpRequest and returns the combined LogQL query. +// This is used only to keep regexp query string support until it gets fully deprecated. +func parseRegexQuery(httpRequest *http.Request) (string, error) { + query := httpRequest.Form.Get("query") + regexp := httpRequest.Form.Get("regexp") + if regexp != "" { + expr, err := syntax.ParseLogSelector(query, true) + if err != nil { + return "", err + } + newExpr, err := syntax.AddFilterExpr(expr, labels.MatchRegexp, "", regexp) + if err != nil { + return "", err + } + query = newExpr.String() + } + return query, nil +} diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go index 287fd6a312..c0447ea115 100644 --- a/pkg/loghttp/query.go +++ b/pkg/loghttp/query.go @@ -12,9 +12,14 @@ import ( json "github.com/json-iterator/go" "github.com/prometheus/common/model" + "github.com/grafana/dskit/httpgrpc" + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logql/syntax" + "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/logqlmodel/stats" "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" + "github.com/grafana/loki/pkg/util" ) var ( @@ -400,6 +405,24 @@ type RangeQuery struct { Shards []string } +func NewRangeQueryWithDefaults() *RangeQuery { + start, end, _ := determineBounds(time.Now(), "", "", "") + result := &RangeQuery{ + Start: start, + End: end, + Limit: defaultQueryLimit, + Direction: defaultDirection, + Interval: 0, + } + result.UpdateStep() + return result +} + +// UpdateStep will adjust the step given new start and end. +func (q *RangeQuery) UpdateStep() { + q.Step = time.Duration(defaultQueryRangeStep(q.Start, q.End)) * time.Second +} + // ParseRangeQuery parses a RangeQuery request from an http request. func ParseRangeQuery(r *http.Request) (*RangeQuery, error) { var result RangeQuery @@ -451,6 +474,23 @@ func ParseRangeQuery(r *http.Request) (*RangeQuery, error) { return nil, errNegativeInterval } + if GetVersion(r.URL.Path) == VersionLegacy { + result.Query, err = parseRegexQuery(r) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + + expr, err := syntax.ParseExpr(result.Query) + if err != nil { + return nil, err + } + + // short circuit metric queries + if _, ok := expr.(syntax.SampleExpr); ok { + return nil, httpgrpc.Errorf(http.StatusBadRequest, "legacy endpoints only support %s result type", logqlmodel.ValueTypeStreams) + } + } + return &result, nil } @@ -460,6 +500,27 @@ func ParseIndexStatsQuery(r *http.Request) (*RangeQuery, error) { return ParseRangeQuery(r) } +func NewVolumeRangeQueryWithDefaults(matchers string) *logproto.VolumeRequest { + start, end, _ := determineBounds(time.Now(), "", "", "") + step := (time.Duration(defaultQueryRangeStep(start, end)) * time.Second).Milliseconds() + from, through := util.RoundToMilliseconds(start, end) + return &logproto.VolumeRequest{ + From: from, + Through: through, + Matchers: matchers, + Limit: seriesvolume.DefaultLimit, + Step: step, + TargetLabels: nil, + AggregateBy: seriesvolume.DefaultAggregateBy, + } +} + +func NewVolumeInstantQueryWithDefaults(matchers string) *logproto.VolumeRequest { + r := NewVolumeRangeQueryWithDefaults(matchers) + r.Step = 0 + return r +} + type VolumeInstantQuery struct { Start time.Time End time.Time diff --git a/pkg/loghttp/tail.go b/pkg/loghttp/tail.go index 4fdbae1b0a..6b9b5ad7d1 100644 --- a/pkg/loghttp/tail.go +++ b/pkg/loghttp/tail.go @@ -8,6 +8,8 @@ import ( json "github.com/json-iterator/go" + "github.com/grafana/dskit/httpgrpc" + "github.com/grafana/loki/pkg/logproto" ) @@ -69,6 +71,11 @@ func ParseTailQuery(r *http.Request) (*logproto.TailRequest, error) { Query: query(r), } + req.Query, err = parseRegexQuery(r) + if err != nil { + return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + req.Limit, err = limit(r) if err != nil { return nil, err diff --git a/pkg/logproto/compat.go b/pkg/logproto/compat.go index f1703e43c0..e195af9687 100644 --- a/pkg/logproto/compat.go +++ b/pkg/logproto/compat.go @@ -319,3 +319,7 @@ func (m *VolumeRequest) LogToSpan(sp opentracing.Span) { otlog.String("end", timestamp.Time(m.GetEnd()).String()), ) } + +func (*VolumeResponse) GetHeaders() []*definitions.PrometheusResponseHeader { + return nil +} diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 81c69f3740..e71e4b5f20 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -323,6 +323,8 @@ type Loki struct { deleteClientMetrics *deletion.DeleteRequestClientMetrics HTTPAuthMiddleware middleware.Interface + + Codec worker.GRPCCodec } // New makes a new Loki. diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go index d6bed1ce8a..d58b7c2c5e 100644 --- a/pkg/loki/modules.go +++ b/pkg/loki/modules.go @@ -374,14 +374,19 @@ func (t *Loki) initQuerier() (services.Service, error) { toMerge := []middleware.Interface{ httpreq.ExtractQueryMetricsMiddleware(), + httpreq.ExtractQueryTagsMiddleware(), + serverutil.RecoveryHTTPMiddleware, + t.HTTPAuthMiddleware, + serverutil.NewPrepopulateMiddleware(), + serverutil.ResponseJSONMiddleware(), } logger := log.With(util_log.Logger, "component", "querier") t.querierAPI = querier.NewQuerierAPI(t.Cfg.Querier, t.Querier, t.Overrides, logger) - indexStatsHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.IndexStats", t.querierAPI) - volumeHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.VolumeInstant", t.querierAPI) - volumeRangeHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.VolumeRange", t.querierAPI) + indexStatsHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.IndexStats", t.Overrides) + volumeHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.VolumeInstant", t.Overrides) + volumeRangeHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.VolumeRange", t.Overrides) if t.supportIndexDeleteRequest() && t.Cfg.CompactorConfig.RetentionEnabled { toMerge = append( @@ -405,7 +410,7 @@ func (t *Loki) initQuerier() (services.Service, error) { ) } - labelsHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.Label", t.querierAPI) + labelsHTTPMiddleware := querier.WrapQuerySpanAndTimeout("query.Label", t.Overrides) if t.Cfg.Querier.PerRequestLimitsEnabled { toMerge = append( @@ -420,34 +425,58 @@ func (t *Loki) initQuerier() (services.Service, error) { httpMiddleware := middleware.Merge(toMerge...) - queryHandlers := map[string]http.Handler{ - "/loki/api/v1/query_range": middleware.Merge( - httpMiddleware, - querier.WrapQuerySpanAndTimeout("query.RangeQuery", t.querierAPI), - ).Wrap(http.HandlerFunc(t.querierAPI.RangeQueryHandler)), + handler := querier.NewQuerierHandler(t.querierAPI) + httpHandler := querier.NewQuerierHTTPHandler(handler) + + // If the querier is running standalone without the query-frontend or query-scheduler, we must register the internal + // HTTP handler externally (as it's the only handler that needs to register on querier routes) and provide the + // external Loki Server HTTP handler to the frontend worker to ensure requests it processes use the default + // middleware instrumentation. + if querierWorkerServiceConfig.QuerierRunningStandalone() { + labelsHTTPMiddleware = middleware.Merge(httpMiddleware, labelsHTTPMiddleware) + indexStatsHTTPMiddleware = middleware.Merge(httpMiddleware, indexStatsHTTPMiddleware) + volumeHTTPMiddleware = middleware.Merge(httpMiddleware, volumeHTTPMiddleware) + volumeRangeHTTPMiddleware = middleware.Merge(httpMiddleware, volumeRangeHTTPMiddleware) + + // First, register the internal querier handler with the external HTTP server + router := t.Server.HTTP + if t.Cfg.Server.PathPrefix != "" { + router = router.PathPrefix(t.Cfg.Server.PathPrefix).Subrouter() + } + + router.Path("/loki/api/v1/query_range").Methods("GET", "POST").Handler( + middleware.Merge( + httpMiddleware, + querier.WrapQuerySpanAndTimeout("query.RangeQuery", t.Overrides), + ).Wrap(httpHandler), + ) - "/loki/api/v1/query": middleware.Merge( - httpMiddleware, - querier.WrapQuerySpanAndTimeout("query.InstantQuery", t.querierAPI), - ).Wrap(http.HandlerFunc(t.querierAPI.InstantQueryHandler)), + router.Path("/loki/api/v1/query").Methods("GET", "POST").Handler( + middleware.Merge( + httpMiddleware, + querier.WrapQuerySpanAndTimeout("query.InstantQuery", t.Overrides), + ).Wrap(httpHandler), + ) - "/loki/api/v1/label": labelsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.LabelHandler)), - "/loki/api/v1/labels": labelsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.LabelHandler)), - "/loki/api/v1/label/{name}/values": labelsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.LabelHandler)), + router.Path("/loki/api/v1/label").Methods("GET", "POST").Handler(labelsHTTPMiddleware.Wrap(httpHandler)) + router.Path("/loki/api/v1/labels").Methods("GET", "POST").Handler(labelsHTTPMiddleware.Wrap(httpHandler)) + router.Path("/loki/api/v1/label/{name}/values").Methods("GET", "POST").Handler(labelsHTTPMiddleware.Wrap(httpHandler)) - "/loki/api/v1/series": querier.WrapQuerySpanAndTimeout("query.Series", t.querierAPI).Wrap(http.HandlerFunc(t.querierAPI.SeriesHandler)), - "/loki/api/v1/index/stats": indexStatsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.IndexStatsHandler)), - "/loki/api/v1/index/volume": volumeHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.VolumeInstantHandler)), - "/loki/api/v1/index/volume_range": volumeRangeHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.VolumeRangeHandler)), + router.Path("/loki/api/v1/series").Methods("GET", "POST").Handler(querier.WrapQuerySpanAndTimeout("query.Series", t.Overrides).Wrap(httpHandler)) + router.Path("/loki/api/v1/index/stats").Methods("GET", "POST").Handler(indexStatsHTTPMiddleware.Wrap(httpHandler)) + router.Path("/loki/api/v1/index/volume").Methods("GET", "POST").Handler(volumeHTTPMiddleware.Wrap(httpHandler)) + router.Path("/loki/api/v1/index/volume_range").Methods("GET", "POST").Handler(volumeRangeHTTPMiddleware.Wrap(httpHandler)) - "/api/prom/query": middleware.Merge( - httpMiddleware, - querier.WrapQuerySpanAndTimeout("query.LogQuery", t.querierAPI), - ).Wrap(http.HandlerFunc(t.querierAPI.LogQueryHandler)), + router.Path("/api/prom/query").Methods("GET", "POST").Handler( + middleware.Merge( + httpMiddleware, + querier.WrapQuerySpanAndTimeout("query.LogQuery", t.Overrides), + ).Wrap(httpHandler), + ) - "/api/prom/label": labelsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.LabelHandler)), - "/api/prom/label/{name}/values": labelsHTTPMiddleware.Wrap(http.HandlerFunc(t.querierAPI.LabelHandler)), - "/api/prom/series": querier.WrapQuerySpanAndTimeout("query.Series", t.querierAPI).Wrap(http.HandlerFunc(t.querierAPI.SeriesHandler)), + router.Path("/api/prom/label").Methods("GET", "POST").Handler(labelsHTTPMiddleware.Wrap(httpHandler)) + router.Path("/api/prom/label/{name}/values").Methods("GET", "POST").Handler(labelsHTTPMiddleware.Wrap(httpHandler)) + router.Path("/api/prom/series").Methods("GET", "POST").Handler(querier.WrapQuerySpanAndTimeout("query.Series", t.Overrides).Wrap(httpHandler)) } // We always want to register tail routes externally, tail requests are different from normal queries, they @@ -459,20 +488,19 @@ func (t *Loki) initQuerier() (services.Service, error) { // is standalone ALL routes are registered externally, and when it's in the same process as a frontend, // we disable the proxying of the tail routes in initQueryFrontend() and we still want these routes regiestered // on the external router. - alwaysExternalHandlers := map[string]http.Handler{ - "/loki/api/v1/tail": http.HandlerFunc(t.querierAPI.TailHandler), - "/api/prom/tail": http.HandlerFunc(t.querierAPI.TailHandler), + t.Server.HTTP.Path("/loki/api/v1/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler))) + t.Server.HTTP.Path("/api/prom/tail").Methods("GET", "POST").Handler(httpMiddleware.Wrap(http.HandlerFunc(t.querierAPI.TailHandler))) + + // Default codec + if t.Codec == nil { + t.Codec = queryrange.DefaultCodec } svc, err := querier.InitWorkerService( querierWorkerServiceConfig, prometheus.DefaultRegisterer, - t.Cfg.Server.PathPrefix, - queryHandlers, - alwaysExternalHandlers, - t.Server.HTTP, - t.Server.HTTPServer.Handler, - t.HTTPAuthMiddleware, + handler, + t.Codec, ) if err != nil { return nil, err diff --git a/pkg/lokifrontend/frontend/v1/frontend_test.go b/pkg/lokifrontend/frontend/v1/frontend_test.go index 1ac564eb34..82422b79b5 100644 --- a/pkg/lokifrontend/frontend/v1/frontend_test.go +++ b/pkg/lokifrontend/frontend/v1/frontend_test.go @@ -13,7 +13,6 @@ import ( "github.com/go-kit/log" "github.com/gorilla/mux" "github.com/grafana/dskit/flagext" - httpgrpc_server "github.com/grafana/dskit/httpgrpc/server" "github.com/grafana/dskit/middleware" "github.com/grafana/dskit/services" "github.com/grafana/dskit/user" @@ -31,6 +30,8 @@ import ( "github.com/grafana/loki/pkg/lokifrontend/frontend/transport" "github.com/grafana/loki/pkg/lokifrontend/frontend/v1/frontendv1pb" + "github.com/grafana/loki/pkg/querier/queryrange" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" querier_worker "github.com/grafana/loki/pkg/querier/worker" "github.com/grafana/loki/pkg/scheduler/queue" ) @@ -38,15 +39,15 @@ import ( const ( query = "/api/v1/query_range?end=1536716898&query=sum%28container_memory_rss%29+by+%28namespace%29&start=1536673680&step=120" responseBody = `{"status":"success","data":{"resultType":"Matrix","result":[{"metric":{"foo":"bar"},"values":[[1536673680,"137"],[1536673780,"137"]]}]}}` + labelQuery = `/api/v1/label/foo/values` ) func TestFrontend(t *testing.T) { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte("Hello World")) - require.NoError(t, err) + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + return &queryrange.LokiLabelNamesResponse{Data: []string{"Hello", "world"}}, nil }) test := func(addr string, _ *Frontend) { - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/", addr), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s", addr, labelQuery), nil) require.NoError(t, err) err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(context.Background(), "1"), req) require.NoError(t, err) @@ -55,10 +56,11 @@ func TestFrontend(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, resp.StatusCode) + defer resp.Body.Close() body, err := io.ReadAll(resp.Body) require.NoError(t, err) - assert.Equal(t, "Hello World", string(body)) + assert.JSONEq(t, `{"values":["Hello", "world"]}`, string(body)) } testFrontend(t, defaultFrontendConfig(), handler, test, false, nil) @@ -72,23 +74,22 @@ func TestFrontendPropagateTrace(t *testing.T) { observedTraceID := make(chan string, 2) - handler := middleware.Tracer{}.Wrap(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - sp := opentracing.SpanFromContext(r.Context()) + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + sp := opentracing.SpanFromContext(ctx) defer sp.Finish() traceID := fmt.Sprintf("%v", sp.Context().(jaeger.SpanContext).TraceID()) observedTraceID <- traceID - _, err = w.Write([]byte(responseBody)) - require.NoError(t, err) - })) + return &queryrange.LokiLabelNamesResponse{Data: []string{"Hello", "world"}}, nil + }) test := func(addr string, _ *Frontend) { sp, ctx := opentracing.StartSpanFromContext(context.Background(), "client") defer sp.Finish() traceID := fmt.Sprintf("%v", sp.Context().(jaeger.SpanContext).TraceID()) - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s", addr, query), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s", addr, labelQuery), nil) require.NoError(t, err) req = req.WithContext(ctx) err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(ctx, "1"), req) @@ -105,12 +106,15 @@ func TestFrontendPropagateTrace(t *testing.T) { require.Equal(t, 200, resp.StatusCode) defer resp.Body.Close() - _, err = io.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) + assert.JSONEq(t, `{"values":["Hello", "world"]}`, string(body)) + // Query should do one call. assert.Equal(t, traceID, <-observedTraceID) } + testFrontend(t, defaultFrontendConfig(), handler, test, false, nil) testFrontend(t, defaultFrontendConfig(), handler, test, true, nil) } @@ -150,12 +154,13 @@ func TestFrontendCheckReady(t *testing.T) { // the underlying query is correctly cancelled _and not retried_. func TestFrontendCancel(t *testing.T) { var tries atomic.Int32 - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-r.Context().Done() + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + <-ctx.Done() tries.Inc() + return nil, ctx.Err() }) test := func(addr string, _ *Frontend) { - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/", addr), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s", addr, labelQuery), nil) require.NoError(t, err) err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(context.Background(), "1"), req) require.NoError(t, err) @@ -180,16 +185,15 @@ func TestFrontendCancel(t *testing.T) { } func TestFrontendMetricsCleanup(t *testing.T) { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte("Hello World")) - require.NoError(t, err) + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + return &queryrange.LokiLabelNamesResponse{Data: []string{"Hello", "world"}}, nil }) for _, matchMaxConcurrency := range []bool{false, true} { reg := prometheus.NewPedanticRegistry() test := func(addr string, fr *Frontend) { - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/", addr), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/%s", addr, labelQuery), nil) require.NoError(t, err) err = user.InjectOrgIDIntoHTTPRequest(user.InjectOrgID(context.Background(), "1"), req) require.NoError(t, err) @@ -202,7 +206,7 @@ func TestFrontendMetricsCleanup(t *testing.T) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - assert.Equal(t, "Hello World", string(body)) + assert.JSONEq(t, `{"values":["Hello", "world"]}`, string(body)) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_query_frontend_queue_length Number of queries in the queue. @@ -222,7 +226,7 @@ func TestFrontendMetricsCleanup(t *testing.T) { } } -func testFrontend(t *testing.T, config Config, handler http.Handler, test func(addr string, frontend *Frontend), matchMaxConcurrency bool, reg prometheus.Registerer) { +func testFrontend(t *testing.T, config Config, handler queryrangebase.Handler, test func(addr string, frontend *Frontend), matchMaxConcurrency bool, reg prometheus.Registerer) { logger := log.NewNopLogger() var workerConfig querier_worker.Config @@ -272,7 +276,7 @@ func testFrontend(t *testing.T, config Config, handler http.Handler, test func(a go grpcServer.Serve(grpcListen) //nolint:errcheck var worker services.Service - worker, err = querier_worker.NewQuerierWorker(workerConfig, nil, httpgrpc_server.NewServer(handler), logger, nil) + worker, err = querier_worker.NewQuerierWorker(workerConfig, nil, handler, logger, nil, queryrange.DefaultCodec) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), worker)) diff --git a/pkg/querier/handler.go b/pkg/querier/handler.go new file mode 100644 index 0000000000..033776d461 --- /dev/null +++ b/pkg/querier/handler.go @@ -0,0 +1,105 @@ +package querier + +import ( + "context" + "fmt" + "net/http" + + "github.com/opentracing/opentracing-go" + + "github.com/grafana/loki/pkg/loghttp" + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/querier/queryrange" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" +) + +type Handler struct { + api *QuerierAPI +} + +func NewQuerierHandler(api *QuerierAPI) *Handler { + return &Handler{ + api: api, + } +} + +func (h *Handler) Do(ctx context.Context, req queryrangebase.Request) (queryrangebase.Response, error) { + span, ctx := opentracing.StartSpanFromContext(ctx, "queryHandler") + defer span.Finish() + + switch concrete := req.(type) { + case *queryrange.LokiRequest: + res, err := h.api.RangeQueryHandler(ctx, concrete) + if err != nil { + return nil, err + } + + params, err := queryrange.ParamsFromRequest(req) + if err != nil { + return nil, err + } + + return queryrange.ResultToResponse(res, params) + case *queryrange.LokiInstantRequest: + res, err := h.api.InstantQueryHandler(ctx, concrete) + if err != nil { + return nil, err + } + + params, err := queryrange.ParamsFromRequest(req) + if err != nil { + return nil, err + } + + return queryrange.ResultToResponse(res, params) + case *queryrange.LokiSeriesRequest: + request := &logproto.SeriesRequest{ + Start: concrete.StartTs, + End: concrete.EndTs, + Groups: concrete.Match, + Shards: concrete.Shards, + } + result, statResult, err := h.api.SeriesHandler(ctx, request) + if err != nil { + return nil, err + } + + return &queryrange.LokiSeriesResponse{ + Status: "success", + Version: uint32(loghttp.VersionV1), + Data: result.Series, + Statistics: statResult, + }, nil + case *queryrange.LabelRequest: + res, err := h.api.LabelHandler(ctx, &concrete.LabelRequest) + if err != nil { + return nil, err + } + + return &queryrange.LokiLabelNamesResponse{ + Status: "success", + Version: uint32(loghttp.VersionV1), + Data: res.Values, + }, nil + case *logproto.IndexStatsRequest: + request := loghttp.NewRangeQueryWithDefaults() + request.Start = concrete.From.Time() + request.End = concrete.Through.Time() + request.Query = concrete.GetQuery() + request.UpdateStep() + + result, err := h.api.IndexStatsHandler(ctx, request) + if err != nil { + return nil, err + } + return &queryrange.IndexStatsResponse{Response: result}, nil + case *logproto.VolumeRequest: + return h.api.VolumeHandler(ctx, concrete) + default: + return nil, fmt.Errorf("unsupported query type %T", req) + } +} + +func NewQuerierHTTPHandler(h *Handler) http.Handler { + return queryrange.NewSerializeHTTPHandler(h, queryrange.DefaultCodec) +} diff --git a/pkg/querier/http.go b/pkg/querier/http.go index b86c53e4e9..cc2343dd24 100644 --- a/pkg/querier/http.go +++ b/pkg/querier/http.go @@ -13,8 +13,6 @@ import ( "github.com/grafana/dskit/middleware" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" "github.com/grafana/dskit/tenant" @@ -69,147 +67,44 @@ func NewQuerierAPI(cfg Config, querier Querier, limits Limits, logger log.Logger } } -// RangeQueryHandler is a http.HandlerFunc for range queries. -func (q *QuerierAPI) RangeQueryHandler(w http.ResponseWriter, r *http.Request) { - request, err := loghttp.ParseRangeQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return +// RangeQueryHandler is a http.HandlerFunc for range queries and legacy log queries +func (q *QuerierAPI) RangeQueryHandler(ctx context.Context, req *queryrange.LokiRequest) (logqlmodel.Result, error) { + if err := q.validateMaxEntriesLimits(ctx, req.Query, req.Limit); err != nil { + return logqlmodel.Result{}, err } - ctx := r.Context() - if err := q.validateMaxEntriesLimits(ctx, request.Query, request.Limit); err != nil { - serverutil.WriteError(err, w) - return - } - - params := logql.NewLiteralParams( - request.Query, - request.Start, - request.End, - request.Step, - request.Interval, - request.Direction, - request.Limit, - request.Shards, - ) - query := q.engine.Query(params) - result, err := query.Exec(ctx) + params, err := queryrange.ParamsFromRequest(req) if err != nil { - serverutil.WriteError(err, w) - return - } - - if err := queryrange.WriteResponse(r, ¶ms, result, w); err != nil { - serverutil.WriteError(err, w) + return logqlmodel.Result{}, err } -} -// InstantQueryHandler is a http.HandlerFunc for instant queries. -func (q *QuerierAPI) InstantQueryHandler(w http.ResponseWriter, r *http.Request) { - request, err := loghttp.ParseInstantQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - - ctx := r.Context() - if err := q.validateMaxEntriesLimits(ctx, request.Query, request.Limit); err != nil { - serverutil.WriteError(err, w) - return - } - - params := logql.NewLiteralParams( - request.Query, - request.Ts, - request.Ts, - 0, - 0, - request.Direction, - request.Limit, - request.Shards, - ) query := q.engine.Query(params) - result, err := query.Exec(ctx) - if err != nil { - serverutil.WriteError(err, w) - return - } - - if err := queryrange.WriteResponse(r, ¶ms, result, w); err != nil { - serverutil.WriteError(err, w) - } + return query.Exec(ctx) } -// LogQueryHandler is a http.HandlerFunc for log only queries. -func (q *QuerierAPI) LogQueryHandler(w http.ResponseWriter, r *http.Request) { - request, err := loghttp.ParseRangeQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - request.Query, err = parseRegexQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return +// InstantQueryHandler is a http.HandlerFunc for instant queries. +func (q *QuerierAPI) InstantQueryHandler(ctx context.Context, req *queryrange.LokiInstantRequest) (logqlmodel.Result, error) { + if err := q.validateMaxEntriesLimits(ctx, req.Query, req.Limit); err != nil { + return logqlmodel.Result{}, err } - expr, err := syntax.ParseExpr(request.Query) + params, err := queryrange.ParamsFromRequest(req) if err != nil { - serverutil.WriteError(err, w) - return - } - - // short circuit metric queries - if _, ok := expr.(syntax.SampleExpr); ok { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, "legacy endpoints only support %s result type", logqlmodel.ValueTypeStreams), w) - return - } - - ctx := r.Context() - if err := q.validateMaxEntriesLimits(ctx, request.Query, request.Limit); err != nil { - serverutil.WriteError(err, w) - return + return logqlmodel.Result{}, err } - - params := logql.NewLiteralParams( - request.Query, - request.Start, - request.End, - request.Step, - request.Interval, - request.Direction, - request.Limit, - request.Shards, - ) query := q.engine.Query(params) - - result, err := query.Exec(ctx) - if err != nil { - serverutil.WriteError(err, w) - return - } - - if err := queryrange.WriteResponse(r, ¶ms, result, w); err != nil { - serverutil.WriteError(err, w) - } + return query.Exec(ctx) } // LabelHandler is a http.HandlerFunc for handling label queries. -func (q *QuerierAPI) LabelHandler(w http.ResponseWriter, r *http.Request) { - req, err := loghttp.ParseLabelQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - +func (q *QuerierAPI) LabelHandler(ctx context.Context, req *logproto.LabelRequest) (*logproto.LabelResponse, error) { timer := prometheus.NewTimer(logql.QueryTime.WithLabelValues("labels")) defer timer.ObserveDuration() start := time.Now() - statsCtx, ctx := stats.NewContext(r.Context()) + statsCtx, ctx := stats.NewContext(ctx) - resp, err := q.querier.Label(r.Context(), req) + resp, err := q.querier.Label(ctx, req) queueTime, _ := ctx.Value(httpreq.QueryQueueTimeHTTPHeader).(time.Duration) resLength := 0 @@ -228,14 +123,7 @@ func (q *QuerierAPI) LabelHandler(w http.ResponseWriter, r *http.Request) { logql.RecordLabelQueryMetrics(ctx, log, *req.Start, *req.End, req.Name, req.Query, strconv.Itoa(status), statResult) - if err != nil { - serverutil.WriteError(err, w) - return - } - - if err := queryrange.WriteResponse(r, nil, resp, w); err != nil { - serverutil.WriteError(err, w) - } + return resp, err } // TailHandler is a http.HandlerFunc for handling tail queries. @@ -251,12 +139,6 @@ func (q *QuerierAPI) TailHandler(w http.ResponseWriter, r *http.Request) { return } - req.Query, err = parseRegexQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - tenantID, err := tenant.TenantID(r.Context()) if err != nil { level.Warn(logger).Log("msg", "error getting tenant id", "err", err) @@ -364,20 +246,14 @@ func (q *QuerierAPI) TailHandler(w http.ResponseWriter, r *http.Request) { // SeriesHandler returns the list of time series that match a certain label set. // See https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers -func (q *QuerierAPI) SeriesHandler(w http.ResponseWriter, r *http.Request) { - req, err := loghttp.ParseAndValidateSeriesQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - +func (q *QuerierAPI) SeriesHandler(ctx context.Context, req *logproto.SeriesRequest) (*logproto.SeriesResponse, stats.Result, error) { timer := prometheus.NewTimer(logql.QueryTime.WithLabelValues("series")) defer timer.ObserveDuration() start := time.Now() - statsCtx, ctx := stats.NewContext(r.Context()) + statsCtx, ctx := stats.NewContext(ctx) - resp, err := q.querier.Series(r.Context(), req) + resp, err := q.querier.Series(ctx, req) queueTime, _ := ctx.Value(httpreq.QueryQueueTimeHTTPHeader).(time.Duration) resLength := 0 @@ -396,120 +272,37 @@ func (q *QuerierAPI) SeriesHandler(w http.ResponseWriter, r *http.Request) { } logql.RecordSeriesQueryMetrics(ctx, log, req.Start, req.End, req.Groups, strconv.Itoa(status), statResult) - if err != nil { - serverutil.WriteError(err, w) - return - } - if err := queryrange.WriteResponse(r, nil, resp, w); err != nil { - serverutil.WriteError(err, w) - } + return resp, statResult, err } // IndexStatsHandler queries the index for the data statistics related to a query -func (q *QuerierAPI) IndexStatsHandler(w http.ResponseWriter, r *http.Request) { - req, err := loghttp.ParseIndexStatsQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - +func (q *QuerierAPI) IndexStatsHandler(ctx context.Context, req *loghttp.RangeQuery) (*logproto.IndexStatsResponse, error) { + // TODO(karsten): we might want to change IndexStats to receive a logproto.IndexStatsRequest instead // TODO(owen-d): log metadata, record stats? - resp, err := q.querier.IndexStats(r.Context(), req) + resp, err := q.querier.IndexStats(ctx, req) if resp == nil { // Some stores don't implement this resp = &index_stats.Stats{} } - if err != nil { - serverutil.WriteError(err, w) - return - } - - if err := queryrange.WriteResponse(r, nil, resp, w); err != nil { - serverutil.WriteError(err, w) - } + return resp, err } //TODO(trevorwhitney): add test for the handler split -// VolumeRangeHandler queries the index label volumes related to the passed matchers and given time range. -// Returns N values where N is the time range / step. -func (q *QuerierAPI) VolumeRangeHandler(w http.ResponseWriter, r *http.Request) { - rawReq, err := loghttp.ParseVolumeRangeQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - - req := &logproto.VolumeRequest{ - From: model.TimeFromUnixNano(rawReq.Start.UnixNano()), - Through: model.TimeFromUnixNano(rawReq.End.UnixNano()), - Matchers: rawReq.Query, - Step: rawReq.Step.Milliseconds(), - Limit: int32(rawReq.Limit), - TargetLabels: rawReq.TargetLabels, - AggregateBy: rawReq.AggregateBy, - } - - q.seriesVolumeHandler(r.Context(), r, req, w) -} - -// VolumeInstantHandler queries the index label volumes related to the passed matchers and given time range. -// Returns a single value for the time range. -func (q *QuerierAPI) VolumeInstantHandler(w http.ResponseWriter, r *http.Request) { - rawReq, err := loghttp.ParseVolumeInstantQuery(r) - if err != nil { - serverutil.WriteError(httpgrpc.Errorf(http.StatusBadRequest, err.Error()), w) - return - } - - req := &logproto.VolumeRequest{ - From: model.TimeFromUnixNano(rawReq.Start.UnixNano()), - Through: model.TimeFromUnixNano(rawReq.End.UnixNano()), - Matchers: rawReq.Query, - Step: 0, - Limit: int32(rawReq.Limit), - TargetLabels: rawReq.TargetLabels, - AggregateBy: rawReq.AggregateBy, - } - - q.seriesVolumeHandler(r.Context(), r, req, w) -} - -func (q *QuerierAPI) seriesVolumeHandler(ctx context.Context, r *http.Request, req *logproto.VolumeRequest, w http.ResponseWriter) { +// VolumeHandler queries the index label volumes related to the passed matchers and given time range. +// Returns either N values where N is the time range / step and a single value for a time range depending on the request. +func (q *QuerierAPI) VolumeHandler(ctx context.Context, req *logproto.VolumeRequest) (*logproto.VolumeResponse, error) { resp, err := q.querier.Volume(ctx, req) if err != nil { - serverutil.WriteError(err, w) - return + return nil, err } - if resp == nil { // Some stores don't implement this - resp = &logproto.VolumeResponse{Volumes: []logproto.Volume{}} + return &logproto.VolumeResponse{Volumes: []logproto.Volume{}}, nil } - if err := queryrange.WriteResponse(r, nil, resp, w); err != nil { - serverutil.WriteError(err, w) - } -} - -// parseRegexQuery parses regex and query querystring from httpRequest and returns the combined LogQL query. -// This is used only to keep regexp query string support until it gets fully deprecated. -func parseRegexQuery(httpRequest *http.Request) (string, error) { - query := httpRequest.Form.Get("query") - regexp := httpRequest.Form.Get("regexp") - if regexp != "" { - expr, err := syntax.ParseLogSelector(query, true) - if err != nil { - return "", err - } - newExpr, err := syntax.AddFilterExpr(expr, labels.MatchRegexp, "", regexp) - if err != nil { - return "", err - } - query = newExpr.String() - } - return query, nil + return resp, nil } func (q *QuerierAPI) validateMaxEntriesLimits(ctx context.Context, query string, limit uint32) error { @@ -540,7 +333,7 @@ func (q *QuerierAPI) validateMaxEntriesLimits(ctx context.Context, query string, // WrapQuerySpanAndTimeout applies a context deadline and a span logger to a query call. // // The timeout is based on the per-tenant query timeout configuration. -func WrapQuerySpanAndTimeout(call string, q *QuerierAPI) middleware.Interface { +func WrapQuerySpanAndTimeout(call string, limits Limits) middleware.Interface { return middleware.Func(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { sp, ctx := opentracing.StartSpanFromContext(req.Context(), call) @@ -555,7 +348,7 @@ func WrapQuerySpanAndTimeout(call string, q *QuerierAPI) middleware.Interface { return } - timeoutCapture := func(id string) time.Duration { return q.limits.QueryTimeout(ctx, id) } + timeoutCapture := func(id string) time.Duration { return limits.QueryTimeout(ctx, id) } timeout := util_validation.SmallestPositiveNonZeroDurationPerTenant(tenants, timeoutCapture) newCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() diff --git a/pkg/querier/http_test.go b/pkg/querier/http_test.go index 7e02dec3fe..5b121ad891 100644 --- a/pkg/querier/http_test.go +++ b/pkg/querier/http_test.go @@ -5,16 +5,14 @@ import ( "fmt" "net/http" "net/http/httptest" - "strings" "testing" "time" "github.com/pkg/errors" "github.com/stretchr/testify/mock" + "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/logqlmodel" - "github.com/grafana/loki/pkg/logqlmodel/stats" "github.com/grafana/loki/pkg/validation" "github.com/go-kit/log" @@ -24,163 +22,6 @@ import ( "github.com/stretchr/testify/require" ) -var ( - statsResultString = `"stats" : { - "ingester" : { - "store": { - "chunk":{ - "compressedBytes": 1, - "decompressedBytes": 2, - "decompressedLines": 3, - "decompressedStructuredMetadataBytes": 0, - "headChunkBytes": 4, - "headChunkLines": 5, - "headChunkStructuredMetadataBytes": 0, - "postFilterLines": 0, - "totalDuplicates": 8 - }, - "chunksDownloadTime": 0, - "totalChunksRef": 0, - "totalChunksDownloaded": 0, - "chunkRefsFetchTime": 0 - }, - "totalBatches": 6, - "totalChunksMatched": 7, - "totalLinesSent": 9, - "totalReached": 10 - }, - "querier": { - "store" : { - "chunk": { - "compressedBytes": 11, - "decompressedBytes": 12, - "decompressedLines": 13, - "decompressedStructuredMetadataBytes": 0, - "headChunkBytes": 14, - "headChunkLines": 15, - "headChunkStructuredMetadataBytes": 0, - "postFilterLines": 0, - "totalDuplicates": 19 - }, - "chunksDownloadTime": 16, - "totalChunksRef": 17, - "totalChunksDownloaded": 18, - "chunkRefsFetchTime": 0 - } - }, - "cache": { - "chunk": { - "entriesFound": 0, - "entriesRequested": 0, - "entriesStored": 0, - "bytesReceived": 0, - "bytesSent": 0, - "requests": 0, - "downloadTime": 0 - }, - "index": { - "entriesFound": 0, - "entriesRequested": 0, - "entriesStored": 0, - "bytesReceived": 0, - "bytesSent": 0, - "requests": 0, - "downloadTime": 0 - }, - "statsResult": { - "entriesFound": 0, - "entriesRequested": 0, - "entriesStored": 0, - "bytesReceived": 0, - "bytesSent": 0, - "requests": 0, - "downloadTime": 0 - }, - "volumeResult": { - "entriesFound": 0, - "entriesRequested": 0, - "entriesStored": 0, - "bytesReceived": 0, - "bytesSent": 0, - "requests": 0, - "downloadTime": 0 - }, - "result": { - "entriesFound": 0, - "entriesRequested": 0, - "entriesStored": 0, - "bytesReceived": 0, - "bytesSent": 0, - "requests": 0, - "downloadTime": 0 - } - }, - "summary": { - "bytesProcessedPerSecond": 20, - "execTime": 22, - "linesProcessedPerSecond": 23, - "queueTime": 21, - "shards": 0, - "splits": 0, - "subqueries": 0, - "totalBytesProcessed": 24, - "totalEntriesReturned": 10, - "totalLinesProcessed": 25, - "totalStructuredMetadataBytesProcessed": 0, - "totalPostFilterLines": 0 - } - }` - statsResult = stats.Result{ - Summary: stats.Summary{ - BytesProcessedPerSecond: 20, - QueueTime: 21, - ExecTime: 22, - LinesProcessedPerSecond: 23, - TotalBytesProcessed: 24, - TotalLinesProcessed: 25, - TotalEntriesReturned: 10, - }, - Querier: stats.Querier{ - Store: stats.Store{ - Chunk: stats.Chunk{ - CompressedBytes: 11, - DecompressedBytes: 12, - DecompressedLines: 13, - HeadChunkBytes: 14, - HeadChunkLines: 15, - TotalDuplicates: 19, - }, - ChunksDownloadTime: 16, - TotalChunksRef: 17, - TotalChunksDownloaded: 18, - }, - }, - - Ingester: stats.Ingester{ - Store: stats.Store{ - Chunk: stats.Chunk{ - CompressedBytes: 1, - DecompressedBytes: 2, - DecompressedLines: 3, - HeadChunkBytes: 4, - HeadChunkLines: 5, - TotalDuplicates: 8, - }, - }, - TotalBatches: 6, - TotalChunksMatched: 7, - TotalLinesSent: 9, - TotalReached: 10, - }, - - Caches: stats.Caches{ - Chunk: stats.Cache{}, - Index: stats.Cache{}, - Result: stats.Cache{}, - }, - } -) - func TestTailHandler(t *testing.T) { tenant.WithDefaultResolver(tenant.NewMultiResolver()) @@ -239,7 +80,6 @@ func TestQueryWrapperMiddleware(t *testing.T) { limits, err := validation.NewOverrides(defaultLimits, nil) require.NoError(t, err) - api := NewQuerierAPI(mockQuerierConfig(), nil, limits, log.NewNopLogger()) // request timeout is 5ms but it sleeps for 100ms, so timeout injected in the request is expected. connSimulator := &slowConnectionSimulator{ @@ -247,7 +87,7 @@ func TestQueryWrapperMiddleware(t *testing.T) { deadline: shortestTimeout, } - midl := WrapQuerySpanAndTimeout("mycall", api).Wrap(connSimulator) + midl := WrapQuerySpanAndTimeout("mycall", limits).Wrap(connSimulator) req, err := http.NewRequest("GET", "/loki/api/v1/label", nil) ctx, cancelFunc := context.WithTimeout(user.InjectOrgID(req.Context(), "fake"), shortestTimeout) @@ -279,14 +119,13 @@ func TestQueryWrapperMiddleware(t *testing.T) { limits, err := validation.NewOverrides(defaultLimits, nil) require.NoError(t, err) - api := NewQuerierAPI(mockQuerierConfig(), nil, limits, log.NewNopLogger()) connSimulator := &slowConnectionSimulator{ sleepFor: time.Millisecond * 100, deadline: shortestTimeout, } - midl := WrapQuerySpanAndTimeout("mycall", api).Wrap(connSimulator) + midl := WrapQuerySpanAndTimeout("mycall", limits).Wrap(connSimulator) req, err := http.NewRequest("GET", "/loki/api/v1/label", nil) ctx, cancelFunc := context.WithTimeout(user.InjectOrgID(req.Context(), "fake"), time.Millisecond*100) @@ -333,16 +172,17 @@ func TestSeriesHandler(t *testing.T) { } expected := `{"status":"success","data":[{"a":"1","b":"2"},{"c":"3","d":"4"}]}` - querier := newQuerierMock() - querier.On("Series", mock.Anything, mock.Anything).Return(ret, nil) - api := setupAPI(querier) + q := newQuerierMock() + q.On("Series", mock.Anything, mock.Anything).Return(ret, nil) + api := setupAPI(q) + handler := NewQuerierHTTPHandler(NewQuerierHandler(api)) req := httptest.NewRequest(http.MethodGet, "/loki/api/v1/series"+ "?start=0"+ "&end=1"+ "&step=42"+ "&query=%7Bfoo%3D%22bar%22%7D", nil) - res := makeRequest(t, api.SeriesHandler, req) + res := makeRequest(t, handler, req) require.Equalf(t, 200, res.Code, "response was not HTTP OK: %s", res.Body.String()) require.JSONEq(t, expected, res.Body.String()) @@ -357,23 +197,19 @@ func TestVolumeHandler(t *testing.T) { t.Run("shared beavhior between range and instant queries", func(t *testing.T) { for _, tc := range []struct { - mode string - handler func(api *QuerierAPI) http.HandlerFunc + mode string + req *logproto.VolumeRequest }{ - {mode: "instant", handler: func(api *QuerierAPI) http.HandlerFunc { return api.VolumeInstantHandler }}, - {mode: "range", handler: func(api *QuerierAPI) http.HandlerFunc { return api.VolumeRangeHandler }}, + {mode: "instant", req: loghttp.NewVolumeInstantQueryWithDefaults(`{foo="bar"}`)}, + {mode: "range", req: loghttp.NewVolumeRangeQueryWithDefaults(`{foo="bar"}`)}, } { t.Run(fmt.Sprintf("%s queries return label volumes from the querier", tc.mode), func(t *testing.T) { querier := newQuerierMock() querier.On("Volume", mock.Anything, mock.Anything).Return(ret, nil) api := setupAPI(querier) - req := httptest.NewRequest(http.MethodGet, "/volume"+ - "?start=0"+ - "&end=1"+ - "&query=%7Bfoo%3D%22bar%22%7D", nil) - - w := makeRequest(t, tc.handler(api), req) + res, err := api.VolumeHandler(context.Background(), tc.req) + require.NoError(t, err) calls := querier.GetMockedCallsByMethod("Volume") require.Len(t, calls, 1) @@ -382,12 +218,7 @@ func TestVolumeHandler(t *testing.T) { require.Equal(t, `{foo="bar"}`, request.Matchers) require.Equal(t, "series", request.AggregateBy) - require.Equal( - t, - `{"volumes":[{"name":"{foo=\"bar\"}","volume":38}]}`, - strings.TrimSpace(w.Body.String()), - ) - require.Equal(t, http.StatusOK, w.Result().StatusCode) + require.Equal(t, ret, res) }) t.Run(fmt.Sprintf("%s queries return nothing when a store doesn't support label volumes", tc.mode), func(t *testing.T) { @@ -395,14 +226,13 @@ func TestVolumeHandler(t *testing.T) { querier.On("Volume", mock.Anything, mock.Anything).Return(nil, nil) api := setupAPI(querier) - req := httptest.NewRequest(http.MethodGet, "/volume?start=0&end=1&query=%7Bfoo%3D%22bar%22%7D", nil) - w := makeRequest(t, tc.handler(api), req) + res, err := api.VolumeHandler(context.Background(), tc.req) + require.NoError(t, err) calls := querier.GetMockedCallsByMethod("Volume") require.Len(t, calls, 1) - require.Equal(t, strings.TrimSpace(w.Body.String()), `{"volumes":[]}`) - require.Equal(t, http.StatusOK, w.Result().StatusCode) + require.Empty(t, res.Volumes) }) t.Run(fmt.Sprintf("%s queries return error when there's an error in the querier", tc.mode), func(t *testing.T) { @@ -412,176 +242,22 @@ func TestVolumeHandler(t *testing.T) { api := setupAPI(querier) - req := httptest.NewRequest(http.MethodGet, "/volume?start=0&end=1&query=%7Bfoo%3D%22bar%22%7D", nil) - w := makeRequest(t, tc.handler(api), req) + _, err = api.VolumeHandler(context.Background(), tc.req) + require.ErrorContains(t, err, "something bad") calls := querier.GetMockedCallsByMethod("Volume") require.Len(t, calls, 1) - - require.Equal(t, strings.TrimSpace(w.Body.String()), `something bad`) - require.Equal(t, http.StatusInternalServerError, w.Result().StatusCode) }) } }) - - t.Run("instant queries set a step of 0", func(t *testing.T) { - querier := newQuerierMock() - querier.On("Volume", mock.Anything, mock.Anything).Return(ret, nil) - api := setupAPI(querier) - - req := httptest.NewRequest(http.MethodGet, "/volume"+ - "?start=0"+ - "&end=1"+ - "&step=42"+ - "&query=%7Bfoo%3D%22bar%22%7D", nil) - makeRequest(t, api.VolumeInstantHandler, req) - - calls := querier.GetMockedCallsByMethod("Volume") - require.Len(t, calls, 1) - - request := calls[0].Arguments[1].(*logproto.VolumeRequest) - require.Equal(t, int64(0), request.Step) - }) - - t.Run("range queries parse step from request", func(t *testing.T) { - querier := newQuerierMock() - querier.On("Volume", mock.Anything, mock.Anything).Return(ret, nil) - api := setupAPI(querier) - - req := httptest.NewRequest(http.MethodGet, "/volume"+ - "?start=0"+ - "&end=1"+ - "&step=42"+ - "&query=%7Bfoo%3D%22bar%22%7D", nil) - makeRequest(t, api.VolumeRangeHandler, req) - - calls := querier.GetMockedCallsByMethod("Volume") - require.Len(t, calls, 1) - - request := calls[0].Arguments[1].(*logproto.VolumeRequest) - require.Equal(t, (42 * time.Second).Milliseconds(), request.Step) - }) - - t.Run("range queries provide default step when not provided", func(t *testing.T) { - querier := newQuerierMock() - querier.On("Volume", mock.Anything, mock.Anything).Return(ret, nil) - api := setupAPI(querier) - - req := httptest.NewRequest(http.MethodGet, "/volume"+ - "?start=0"+ - "&end=1"+ - "&query=%7Bfoo%3D%22bar%22%7D", nil) - makeRequest(t, api.VolumeRangeHandler, req) - - calls := querier.GetMockedCallsByMethod("Volume") - require.Len(t, calls, 1) - - request := calls[0].Arguments[1].(*logproto.VolumeRequest) - require.Equal(t, time.Second.Milliseconds(), request.Step) - }) } -func TestResponseFormat(t *testing.T) { - for _, tc := range []struct { - url string - accept string - handler func(api *QuerierAPI) http.HandlerFunc - result logqlmodel.Result - expectedRespone string - }{ - { - url: "/api/prom/query", - handler: func(api *QuerierAPI) http.HandlerFunc { - return api.LogQueryHandler - }, - result: logqlmodel.Result{ - Data: logqlmodel.Streams{ - logproto.Stream{ - Entries: []logproto.Entry{ - { - Timestamp: time.Unix(0, 123456789012345).UTC(), - Line: "super line", - }, - }, - Labels: `{foo="bar"}`, - }, - }, - Statistics: statsResult, - }, - expectedRespone: `{ - ` + statsResultString + `, - "streams": [ - { - "labels": "{foo=\"bar\"}", - "entries": [ - { - "line": "super line", - "ts": "1970-01-02T10:17:36.789012345Z" - } - ] - } - ] - }`, - }, - { - url: "/loki/api/v1/query_range", - handler: func(api *QuerierAPI) http.HandlerFunc { - return api.RangeQueryHandler - }, - result: logqlmodel.Result{ - Data: logqlmodel.Streams{ - logproto.Stream{ - Entries: []logproto.Entry{ - { - Timestamp: time.Unix(0, 123456789012345).UTC(), - Line: "super line", - }, - }, - Labels: `{foo="bar"}`, - }, - }, - Statistics: statsResult, - }, - expectedRespone: `{ - "status": "success", - "data": { - "resultType": "streams", - ` + statsResultString + `, - "result": [{ - "stream": {"foo": "bar"}, - "values": [ - ["123456789012345", "super line"] - ] - }] - } - }`, - }, - } { - t.Run(fmt.Sprintf("%s returns the expected format", tc.url), func(t *testing.T) { - engine := newEngineMock() - engine.On("Query", mock.Anything, mock.Anything).Return(queryMock{tc.result}) - api := setupAPIWithEngine(engine) - - req := httptest.NewRequest(http.MethodGet, tc.url+ - "?start=0"+ - "&end=1"+ - "&query=%7Bfoo%3D%22bar%22%7D", nil) - req = req.WithContext(user.InjectOrgID(context.Background(), "1")) - - w := makeRequest(t, tc.handler(api), req) - - require.Equalf(t, http.StatusOK, w.Code, "unexpected response: %s", w.Body.String()) - require.JSONEq(t, tc.expectedRespone, w.Body.String()) - }) - } -} - -func makeRequest(t *testing.T, handler http.HandlerFunc, req *http.Request) *httptest.ResponseRecorder { +func makeRequest(t *testing.T, handler http.Handler, req *http.Request) *httptest.ResponseRecorder { err := req.ParseForm() require.NoError(t, err) w := httptest.NewRecorder() - handler(w, req) + handler.ServeHTTP(w, req) return w } @@ -589,10 +265,3 @@ func setupAPI(querier *querierMock) *QuerierAPI { api := NewQuerierAPI(Config{}, querier, nil, log.NewNopLogger()) return api } - -func setupAPIWithEngine(engine *engineMock) *QuerierAPI { - limits, _ := validation.NewOverrides(validation.Limits{}, mockTenantLimits{}) - api := NewQuerierAPI(Config{}, nil, limits, log.NewNopLogger()) - api.engine = engine - return api -} diff --git a/pkg/querier/queryrange/codec.go b/pkg/querier/queryrange/codec.go index 1c0c26c9cf..fdca5a6a5c 100644 --- a/pkg/querier/queryrange/codec.go +++ b/pkg/querier/queryrange/codec.go @@ -9,6 +9,7 @@ import ( io "io" "net/http" "net/url" + "regexp" "sort" strings "strings" "time" @@ -16,6 +17,7 @@ import ( "github.com/grafana/loki/pkg/storage/stores/index/seriesvolume" "github.com/grafana/dskit/httpgrpc" + "github.com/grafana/dskit/user" json "github.com/json-iterator/go" "github.com/opentracing/opentracing-go" otlog "github.com/opentracing/opentracing-go/log" @@ -251,20 +253,21 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer switch op := getOperation(r.URL.Path); op { case QueryRangeOp: - req, err := loghttp.ParseRangeQuery(r) + rangeQuery, err := loghttp.ParseRangeQuery(r) if err != nil { return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) } + return &LokiRequest{ - Query: req.Query, - Limit: req.Limit, - Direction: req.Direction, - StartTs: req.Start.UTC(), - EndTs: req.End.UTC(), - Step: req.Step.Milliseconds(), - Interval: req.Interval.Milliseconds(), + Query: rangeQuery.Query, + Limit: rangeQuery.Limit, + Direction: rangeQuery.Direction, + StartTs: rangeQuery.Start.UTC(), + EndTs: rangeQuery.End.UTC(), + Step: rangeQuery.Step.Milliseconds(), + Interval: rangeQuery.Interval.Milliseconds(), Path: r.URL.Path, - Shards: req.Shards, + Shards: rangeQuery.Shards, }, nil case InstantQueryOp: req, err := loghttp.ParseInstantQuery(r) @@ -342,8 +345,162 @@ func (Codec) DecodeRequest(_ context.Context, r *http.Request, _ []string) (quer AggregateBy: req.AggregateBy, }, err default: - return nil, httpgrpc.Errorf(http.StatusBadRequest, fmt.Sprintf("unknown request path: %s", r.URL.Path)) + return nil, httpgrpc.Errorf(http.StatusNotFound, fmt.Sprintf("unknown request path: %s", r.URL.Path)) + } +} + +// labelNamesRoutes is used to extract the name for querying label values. +var labelNamesRoutes = regexp.MustCompile(`/loki/api/v1/label/(?P[^/]+)/values`) + +// DecodeHTTPGrpcRequest decodes an httpgrp.HTTPrequest to queryrangebase.Request. +func (Codec) DecodeHTTPGrpcRequest(ctx context.Context, r *httpgrpc.HTTPRequest) (queryrangebase.Request, context.Context, error) { + httpReq, err := http.NewRequest(r.Method, r.Url, io.NopCloser(bytes.NewBuffer(r.Body))) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusInternalServerError, err.Error()) + } + httpReq = httpReq.WithContext(ctx) + httpReq.RequestURI = r.Url + httpReq.ContentLength = int64(len(r.Body)) + + // Note that the org ID should be injected by the scheduler processor. + for _, h := range r.Headers { + httpReq.Header[h.Key] = h.Values + } + + // If there is not org ID in the context, we try the HTTP request. + _, err = user.ExtractOrgID(ctx) + if err != nil { + _, ctx, err = user.ExtractOrgIDFromHTTPRequest(httpReq) + if err != nil { + return nil, nil, err + } + } + + if err := httpReq.ParseForm(); err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + + switch op := getOperation(httpReq.URL.Path); op { + case QueryRangeOp: + req, err := loghttp.ParseRangeQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiRequest{ + Query: req.Query, + Limit: req.Limit, + Direction: req.Direction, + StartTs: req.Start.UTC(), + EndTs: req.End.UTC(), + Step: req.Step.Milliseconds(), + Interval: req.Interval.Milliseconds(), + Path: r.Url, + Shards: req.Shards, + }, ctx, nil + case InstantQueryOp: + req, err := loghttp.ParseInstantQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiInstantRequest{ + Query: req.Query, + Limit: req.Limit, + Direction: req.Direction, + TimeTs: req.Ts.UTC(), + Path: r.Url, + Shards: req.Shards, + }, ctx, nil + case SeriesOp: + req, err := loghttp.ParseAndValidateSeriesQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + return &LokiSeriesRequest{ + Match: req.Groups, + StartTs: req.Start.UTC(), + EndTs: req.End.UTC(), + Path: r.Url, + Shards: req.Shards, + }, ctx, nil + case LabelNamesOp: + req, err := loghttp.ParseLabelQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + + if req.Name == "" { + if match := labelNamesRoutes.FindSubmatch([]byte(httpReq.URL.Path)); len(match) > 1 { + req.Name = string(match[1]) + req.Values = true + } + } + + return &LabelRequest{ + LabelRequest: *req, + path: httpReq.URL.Path, + }, ctx, nil + case IndexStatsOp: + req, err := loghttp.ParseIndexStatsQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + from, through := util.RoundToMilliseconds(req.Start, req.End) + return &logproto.IndexStatsRequest{ + From: from, + Through: through, + Matchers: req.Query, + }, ctx, err + case VolumeOp: + req, err := loghttp.ParseVolumeInstantQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + from, through := util.RoundToMilliseconds(req.Start, req.End) + return &logproto.VolumeRequest{ + From: from, + Through: through, + Matchers: req.Query, + Limit: int32(req.Limit), + Step: 0, + TargetLabels: req.TargetLabels, + AggregateBy: req.AggregateBy, + }, ctx, err + case VolumeRangeOp: + req, err := loghttp.ParseVolumeRangeQuery(httpReq) + if err != nil { + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, err.Error()) + } + from, through := util.RoundToMilliseconds(req.Start, req.End) + return &logproto.VolumeRequest{ + From: from, + Through: through, + Matchers: req.Query, + Limit: int32(req.Limit), + Step: req.Step.Milliseconds(), + TargetLabels: req.TargetLabels, + AggregateBy: req.AggregateBy, + }, ctx, err + default: + return nil, ctx, httpgrpc.Errorf(http.StatusBadRequest, fmt.Sprintf("unknown request path: %s", r.Url)) + } +} + +func (Codec) EncodeHTTPGrpcResponse(ctx context.Context, req *httpgrpc.HTTPRequest, res queryrangebase.Response) (*httpgrpc.HTTPResponse, error) { + version := loghttp.GetVersion(req.Url) + var buf bytes.Buffer + + err := encodeResponseJSONTo(version, res, &buf) + if err != nil { + return nil, err } + + return &httpgrpc.HTTPResponse{ + Code: int32(http.StatusOK), + Body: buf.Bytes(), + Headers: []*httpgrpc.Header{ + {Key: "Content-Type", Values: []string{"application/json; charset=UTF-8"}}, + }, + }, nil } func (c Codec) EncodeRequest(ctx context.Context, r queryrangebase.Request) (*http.Request, error) { @@ -622,7 +779,7 @@ func decodeResponseJSON(r *http.Response, req queryrangebase.Request) (queryrang }, nil case loghttp.ResultTypeStream: // This is the same as in querysharding.go - params, err := paramsFromRequest(req) + params, err := ParamsFromRequest(req) if err != nil { return nil, err } @@ -740,9 +897,27 @@ func encodeResponseJSON(ctx context.Context, version loghttp.Version, res queryr defer sp.Finish() var buf bytes.Buffer + err := encodeResponseJSONTo(version, res, &buf) + if err != nil { + return nil, err + } + + sp.LogFields(otlog.Int("bytes", buf.Len())) + + resp := http.Response{ + Header: http.Header{ + "Content-Type": []string{"application/json; charset=UTF-8"}, + }, + Body: io.NopCloser(&buf), + StatusCode: http.StatusOK, + } + return &resp, nil +} + +func encodeResponseJSONTo(version loghttp.Version, res queryrangebase.Response, w io.Writer) error { switch response := res.(type) { case *LokiPromResponse: - return response.encode(ctx) + return response.encodeTo(w) case *LokiResponse: streams := make([]logproto.Stream, len(response.Data.Result)) @@ -752,62 +927,50 @@ func encodeResponseJSON(ctx context.Context, version loghttp.Version, res queryr Entries: stream.Entries, } } - result := logqlmodel.Result{ - Data: logqlmodel.Streams(streams), - Statistics: response.Statistics, - } if version == loghttp.VersionLegacy { - if err := marshal_legacy.WriteQueryResponseJSON(result, &buf); err != nil { - return nil, err + result := logqlmodel.Result{ + Data: logqlmodel.Streams(streams), + Statistics: response.Statistics, + } + if err := marshal_legacy.WriteQueryResponseJSON(result, w); err != nil { + return err } } else { - if err := marshal.WriteQueryResponseJSON(result, &buf); err != nil { - return nil, err + if err := marshal.WriteQueryResponseJSON(logqlmodel.Streams(streams), response.Statistics, w); err != nil { + return err } } case *MergedSeriesResponseView: - if err := WriteSeriesResponseViewJSON(response, &buf); err != nil { - return nil, err + if err := WriteSeriesResponseViewJSON(response, w); err != nil { + return err } case *LokiSeriesResponse: - result := logproto.SeriesResponse{ - Series: response.Data, - } - if err := marshal.WriteSeriesResponseJSON(result, &buf); err != nil { - return nil, err + if err := marshal.WriteSeriesResponseJSON(response.Data, w); err != nil { + return err } case *LokiLabelNamesResponse: if loghttp.Version(response.Version) == loghttp.VersionLegacy { - if err := marshal_legacy.WriteLabelResponseJSON(logproto.LabelResponse{Values: response.Data}, &buf); err != nil { - return nil, err + if err := marshal_legacy.WriteLabelResponseJSON(logproto.LabelResponse{Values: response.Data}, w); err != nil { + return err } } else { - if err := marshal.WriteLabelResponseJSON(logproto.LabelResponse{Values: response.Data}, &buf); err != nil { - return nil, err + if err := marshal.WriteLabelResponseJSON(response.Data, w); err != nil { + return err } } case *IndexStatsResponse: - if err := marshal.WriteIndexStatsResponseJSON(response.Response, &buf); err != nil { - return nil, err + if err := marshal.WriteIndexStatsResponseJSON(response.Response, w); err != nil { + return err } case *VolumeResponse: - if err := marshal.WriteVolumeResponseJSON(response.Response, &buf); err != nil { - return nil, err + if err := marshal.WriteVolumeResponseJSON(response.Response, w); err != nil { + return err } default: - return nil, httpgrpc.Errorf(http.StatusInternalServerError, fmt.Sprintf("invalid response formatt, got (%T)", res)) + return httpgrpc.Errorf(http.StatusInternalServerError, fmt.Sprintf("invalid response format, got (%T)", res)) } - sp.LogFields(otlog.Int("bytes", buf.Len())) - - resp := http.Response{ - Header: http.Header{ - "Content-Type": []string{"application/json; charset=UTF-8"}, - }, - Body: io.NopCloser(&buf), - StatusCode: http.StatusOK, - } - return &resp, nil + return nil } func encodeResponseProtobuf(ctx context.Context, res queryrangebase.Response) (*http.Response, error) { @@ -1149,7 +1312,7 @@ func (res LokiResponse) Count() int64 { return result } -func paramsFromRequest(req queryrangebase.Request) (logql.Params, error) { +func ParamsFromRequest(req queryrangebase.Request) (logql.Params, error) { switch r := req.(type) { case *LokiRequest: return ¶msRangeWrapper{ diff --git a/pkg/querier/queryrange/codec_test.go b/pkg/querier/queryrange/codec_test.go index f792d98e78..9be6a1c856 100644 --- a/pkg/querier/queryrange/codec_test.go +++ b/pkg/querier/queryrange/codec_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" "net/url" "strconv" strings "strings" @@ -176,7 +177,6 @@ func Test_codec_EncodeDecodeRequest(t *testing.T) { if err != nil { t.Fatal(err) } - // TODO: context must save mux vars got, err := DefaultCodec.DecodeRequest(context.TODO(), req, nil) if (err != nil) != tt.wantErr { t.Errorf("codec.DecodeRequest() error = %v, wantErr %v", err, tt.wantErr) @@ -715,6 +715,44 @@ func Test_codec_seriesVolume_EncodeRequest(t *testing.T) { require.Equal(t, `foo,bar`, got.URL.Query().Get("targetLabels")) } +func Test_codec_seriesVolume_DecodeRequest(t *testing.T) { + t.Run("instant queries set a step of 0", func(t *testing.T) { + + req := httptest.NewRequest(http.MethodGet, "/loki/api/v1/index/volume"+ + "?start=0"+ + "&end=1"+ + "&step=42"+ + "&query=%7Bfoo%3D%22bar%22%7D", nil) + got, err := DefaultCodec.DecodeRequest(context.Background(), req, nil) + require.NoError(t, err) + + require.Equal(t, int64(0), got.(*logproto.VolumeRequest).Step) + }) + + t.Run("range queries parse step from request", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/loki/api/v1/index/volume_range"+ + "?start=0"+ + "&end=1"+ + "&step=42"+ + "&query=%7Bfoo%3D%22bar%22%7D", nil) + got, err := DefaultCodec.DecodeRequest(context.Background(), req, nil) + require.NoError(t, err) + + require.Equal(t, (42 * time.Second).Milliseconds(), got.(*logproto.VolumeRequest).Step) + }) + + t.Run("range queries provide default step when not provided", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/loki/api/v1/index/volume_range"+ + "?start=0"+ + "&end=1"+ + "&query=%7Bfoo%3D%22bar%22%7D", nil) + got, err := DefaultCodec.DecodeRequest(context.Background(), req, nil) + require.NoError(t, err) + + require.Equal(t, time.Second.Milliseconds(), got.(*logproto.VolumeRequest).Step) + }) +} + func Test_codec_EncodeResponse(t *testing.T) { tests := []struct { name string diff --git a/pkg/querier/queryrange/limits_test.go b/pkg/querier/queryrange/limits_test.go index 5a11e49979..cca9946f0c 100644 --- a/pkg/querier/queryrange/limits_test.go +++ b/pkg/querier/queryrange/limits_test.go @@ -17,7 +17,7 @@ import ( "gopkg.in/yaml.v2" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logqlmodel/stats" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/storage/config" util_log "github.com/grafana/loki/pkg/util/log" @@ -104,14 +104,14 @@ func Test_seriesLimiter(t *testing.T) { }() // first time returns a single series if *c == 0 { - if err := marshal.WriteQueryResponseJSON(logqlmodel.Result{Data: matrix}, rw); err != nil { + if err := marshal.WriteQueryResponseJSON(matrix, stats.Result{}, rw); err != nil { panic(err) } return } // second time returns a different series. - if err := marshal.WriteQueryResponseJSON(logqlmodel.Result{ - Data: promql.Matrix{ + if err := marshal.WriteQueryResponseJSON( + promql.Matrix{ { Floats: []promql.FPoint{ { @@ -131,7 +131,8 @@ func Test_seriesLimiter(t *testing.T) { }, }, }, - }, rw); err != nil { + stats.Result{}, + rw); err != nil { panic(err) } }) diff --git a/pkg/querier/queryrange/marshal.go b/pkg/querier/queryrange/marshal.go index 7cb8f74094..512cfb321b 100644 --- a/pkg/querier/queryrange/marshal.go +++ b/pkg/querier/queryrange/marshal.go @@ -5,18 +5,14 @@ package queryrange import ( "fmt" "io" - "net/http" "github.com/prometheus/prometheus/promql" "github.com/grafana/loki/pkg/loghttp" - "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logql" "github.com/grafana/loki/pkg/logql/sketch" "github.com/grafana/loki/pkg/logqlmodel" "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" - "github.com/grafana/loki/pkg/storage/stores/index/stats" - "github.com/grafana/loki/pkg/util/marshal" ) const ( @@ -24,83 +20,19 @@ const ( ProtobufType = `application/vnd.google.protobuf` ) -func WriteResponse(req *http.Request, params *logql.LiteralParams, v any, w http.ResponseWriter) error { - if req.Header.Get("Accept") == ProtobufType { - w.Header().Add("Content-Type", ProtobufType) - return WriteResponseProtobuf(req, params, v, w) - } - - w.Header().Add("Content-Type", JSONType) - return marshal.WriteResponseJSON(req, v, w) -} - -func WriteResponseProtobuf(req *http.Request, params *logql.LiteralParams, v any, w http.ResponseWriter) error { - switch result := v.(type) { - case logqlmodel.Result: - return WriteQueryResponseProtobuf(params, result, w) - case *logproto.LabelResponse: - version := loghttp.GetVersion(req.RequestURI) - return WriteLabelResponseProtobuf(version, *result, w) - case *logproto.SeriesResponse: - version := loghttp.GetVersion(req.RequestURI) - return WriteSeriesResponseProtobuf(version, *result, w) - case *stats.Stats: - return WriteIndexStatsResponseProtobuf(result, w) - case *logproto.VolumeResponse: - return WriteVolumeResponseProtobuf(result, w) - } - return fmt.Errorf("unknown response type %T", v) -} - // WriteQueryResponseProtobuf marshals the promql.Value to queryrange QueryResonse and then // writes it to the provided io.Writer. -func WriteQueryResponseProtobuf(params *logql.LiteralParams, v logqlmodel.Result, w io.Writer) error { - p, err := ResultToResponse(v, params) +func WriteQueryResponseProtobuf(params logql.Params, v logqlmodel.Result, w io.Writer) error { + r, err := ResultToResponse(v, params) if err != nil { return err } - buf, err := p.Marshal() - if err != nil { - return err - } - _, err = w.Write(buf) - return err -} - -// WriteLabelResponseProtobuf marshals a logproto.LabelResponse to queryrange LokiLabelNamesResponse -// and then writes it to the provided io.Writer. -func WriteLabelResponseProtobuf(version loghttp.Version, l logproto.LabelResponse, w io.Writer) error { - p := QueryResponse{ - Response: &QueryResponse_Labels{ - Labels: &LokiLabelNamesResponse{ - Status: "success", - Data: l.Values, - Version: uint32(version), - // Statistics: statResult, - }, - }, - } - buf, err := p.Marshal() + p, err := QueryResponseWrap(r) if err != nil { return err } - _, err = w.Write(buf) - return err -} -// WriteSeriesResponseProtobuf marshals a logproto.SeriesResponse to queryrange LokiSeriesResponse -// and then writes it to the provided io.Writer. -func WriteSeriesResponseProtobuf(version loghttp.Version, r logproto.SeriesResponse, w io.Writer) error { - p := QueryResponse{ - Response: &QueryResponse_Series{ - Series: &LokiSeriesResponse{ - Status: "success", - Version: uint32(version), - Data: r.Series, - // Statistics: statResult, - }}, - } buf, err := p.Marshal() if err != nil { return err @@ -109,42 +41,8 @@ func WriteSeriesResponseProtobuf(version loghttp.Version, r logproto.SeriesRespo return err } -// WriteIndexStatsResponseProtobuf marshals a gatewaypb.Stats to queryrange IndexStatsResponse -// and then writes it to the provided io.Writer. -func WriteIndexStatsResponseProtobuf(r *stats.Stats, w io.Writer) error { - p := QueryResponse{ - Response: &QueryResponse_Stats{ - Stats: &IndexStatsResponse{ - Response: r, - }}, - } - buf, err := p.Marshal() - if err != nil { - return err - } - _, err = w.Write(buf) - return err -} - -// WriteIndexStatsResponseProtobuf marshals a logproto.VolumeResponse to queryrange.QueryResponse -// and then writes it to the provided io.Writer. -func WriteVolumeResponseProtobuf(r *logproto.VolumeResponse, w io.Writer) error { - p := QueryResponse{ - Response: &QueryResponse_Volume{ - Volume: &VolumeResponse{ - Response: r, - }}, - } - buf, err := p.Marshal() - if err != nil { - return err - } - _, err = w.Write(buf) - return err -} - -// ResultToResponse is the reverse of ResponseToResult in downstreamer. -func ResultToResponse(result logqlmodel.Result, params *logql.LiteralParams) (*QueryResponse, error) { +// ResultToResponse is the reverse of ResponseToResult below. +func ResultToResponse(result logqlmodel.Result, params logql.Params) (queryrangebase.Response, error) { switch data := result.Data.(type) { case promql.Vector: sampleStream, err := queryrangebase.FromValue(data) @@ -152,38 +50,30 @@ func ResultToResponse(result logqlmodel.Result, params *logql.LiteralParams) (*Q return nil, err } - return &QueryResponse{ - Response: &QueryResponse_Prom{ - Prom: &LokiPromResponse{ - Response: &queryrangebase.PrometheusResponse{ - Status: "success", - Data: queryrangebase.PrometheusData{ - ResultType: loghttp.ResultTypeVector, - Result: sampleStream, - }, - }, - Statistics: result.Statistics, + return &LokiPromResponse{ + Response: &queryrangebase.PrometheusResponse{ + Status: "success", + Data: queryrangebase.PrometheusData{ + ResultType: loghttp.ResultTypeVector, + Result: sampleStream, }, }, + Statistics: result.Statistics, }, nil case promql.Matrix: sampleStream, err := queryrangebase.FromValue(data) if err != nil { return nil, err } - return &QueryResponse{ - Response: &QueryResponse_Prom{ - Prom: &LokiPromResponse{ - Response: &queryrangebase.PrometheusResponse{ - Status: "success", - Data: queryrangebase.PrometheusData{ - ResultType: loghttp.ResultTypeMatrix, - Result: sampleStream, - }, - }, - Statistics: result.Statistics, + return &LokiPromResponse{ + Response: &queryrangebase.PrometheusResponse{ + Status: "success", + Data: queryrangebase.PrometheusData{ + ResultType: loghttp.ResultTypeMatrix, + Result: sampleStream, }, }, + Statistics: result.Statistics, }, nil case promql.Scalar: sampleStream, err := queryrangebase.FromValue(data) @@ -191,53 +81,65 @@ func ResultToResponse(result logqlmodel.Result, params *logql.LiteralParams) (*Q return nil, err } - return &QueryResponse{ - Response: &QueryResponse_Prom{ - Prom: &LokiPromResponse{ - Response: &queryrangebase.PrometheusResponse{ - Status: "success", - Data: queryrangebase.PrometheusData{ - ResultType: loghttp.ResultTypeScalar, - Result: sampleStream, - }, - }, - Statistics: result.Statistics, + return &LokiPromResponse{ + Response: &queryrangebase.PrometheusResponse{ + Status: "success", + Data: queryrangebase.PrometheusData{ + ResultType: loghttp.ResultTypeScalar, + Result: sampleStream, }, }, + Statistics: result.Statistics, }, nil case logqlmodel.Streams: - return &QueryResponse{ - Response: &QueryResponse_Streams{ - // Note: we are omitting the Version here because the Protobuf already defines a schema. - Streams: &LokiResponse{ - Direction: params.Direction(), - Limit: params.Limit(), - Data: LokiData{ - ResultType: loghttp.ResultTypeStream, - Result: data, - }, - Status: "success", - Statistics: result.Statistics, - }, + return &LokiResponse{ + Direction: params.Direction(), + Limit: params.Limit(), + Data: LokiData{ + ResultType: loghttp.ResultTypeStream, + Result: data, }, + Status: "success", + Statistics: result.Statistics, }, nil case sketch.TopKMatrix: sk, err := data.ToProto() - if err != nil { - return nil, err - } - return &QueryResponse{ - Response: &QueryResponse_TopkSketches{ - TopkSketches: &TopKSketchesResponse{Response: sk}, - }, - }, nil + return &TopKSketchesResponse{Response: sk}, err case sketch.QuantileSketchMatrix: - return &QueryResponse{ - Response: &QueryResponse_QuantileSketches{ - QuantileSketches: &QuantileSketchResponse{Response: data.ToProto()}, - }, - }, nil + return &QuantileSketchResponse{Response: data.ToProto()}, nil } return nil, fmt.Errorf("unsupported data type: %t", result.Data) } + +func QueryResponseWrap(res queryrangebase.Response) (*QueryResponse, error) { + p := &QueryResponse{} + + switch response := res.(type) { + case *LokiPromResponse: + p.Response = &QueryResponse_Prom{response} + case *LokiResponse: + p.Response = &QueryResponse_Streams{response} + case *LokiSeriesResponse: + p.Response = &QueryResponse_Series{response} + case *MergedSeriesResponseView: + mat, err := response.Materialize() + if err != nil { + return p, err + } + p.Response = &QueryResponse_Series{mat} + case *LokiLabelNamesResponse: + p.Response = &QueryResponse_Labels{response} + case *IndexStatsResponse: + p.Response = &QueryResponse_Stats{response} + case *TopKSketchesResponse: + p.Response = &QueryResponse_TopkSketches{response} + case *QuantileSketchResponse: + p.Response = &QueryResponse_QuantileSketches{response} + default: + return nil, fmt.Errorf("invalid response format, got (%T)", res) + } + + return p, nil + +} diff --git a/pkg/querier/queryrange/prometheus.go b/pkg/querier/queryrange/prometheus.go index 14fcfdcafd..145eccba7d 100644 --- a/pkg/querier/queryrange/prometheus.go +++ b/pkg/querier/queryrange/prometheus.go @@ -43,6 +43,28 @@ func (PrometheusExtractor) ResponseWithoutHeaders(resp queryrangebase.Response) // encode encodes a Prometheus response and injects Loki stats. func (p *LokiPromResponse) encode(ctx context.Context) (*http.Response, error) { sp := opentracing.SpanFromContext(ctx) + var buf bytes.Buffer + + err := p.encodeTo(&buf) + if err != nil { + return nil, err + } + + if sp != nil { + sp.LogFields(otlog.Int("bytes", buf.Len())) + } + + resp := http.Response{ + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(&buf), + StatusCode: http.StatusOK, + } + return &resp, nil +} + +func (p *LokiPromResponse) encodeTo(w io.Writer) error { var ( b []byte err error @@ -57,21 +79,11 @@ func (p *LokiPromResponse) encode(ctx context.Context) (*http.Response, error) { b, err = p.marshalScalar() } if err != nil { - return nil, err - } - - if sp != nil { - sp.LogFields(otlog.Int("bytes", len(b))) + return err } - resp := http.Response{ - Header: http.Header{ - "Content-Type": []string{"application/json"}, - }, - Body: io.NopCloser(bytes.NewBuffer(b)), - StatusCode: http.StatusOK, - } - return &resp, nil + _, err = w.Write(b) + return err } func (p *LokiPromResponse) marshalVector() ([]byte, error) { diff --git a/pkg/querier/queryrange/queryrange.pb.go b/pkg/querier/queryrange/queryrange.pb.go index da8ee77ddc..29704deef0 100644 --- a/pkg/querier/queryrange/queryrange.pb.go +++ b/pkg/querier/queryrange/queryrange.pb.go @@ -5,6 +5,7 @@ package queryrange import ( fmt "fmt" + _ "github.com/gogo/googleapis/google/rpc" _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" _ "github.com/gogo/protobuf/types" @@ -966,86 +967,87 @@ func init() { } var fileDescriptor_51b9d53b40d11902 = []byte{ - // 1257 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x97, 0xcd, 0x6f, 0x1b, 0xc5, - 0x1b, 0xc7, 0x77, 0xfd, 0x96, 0x78, 0xdc, 0xe6, 0xd7, 0xdf, 0x24, 0x6a, 0x97, 0x50, 0xed, 0x5a, - 0x96, 0xa0, 0x46, 0x02, 0x5b, 0x38, 0xa5, 0xa5, 0xbc, 0x89, 0x2e, 0xa1, 0x72, 0x44, 0x41, 0xed, - 0xc6, 0xe2, 0x3e, 0x8e, 0x27, 0xf6, 0xe2, 0x7d, 0xcb, 0xce, 0x38, 0x6a, 0x6e, 0x5c, 0xb8, 0x81, - 0xd4, 0xbf, 0x02, 0x21, 0x51, 0xf1, 0x07, 0x70, 0xe4, 0x94, 0x63, 0x8e, 0x95, 0x25, 0x16, 0xe2, - 0x5c, 0x20, 0xa7, 0xfc, 0x09, 0x68, 0x66, 0x76, 0xd7, 0xbb, 0xb6, 0x93, 0xda, 0xe5, 0xd2, 0x48, - 0x5c, 0xec, 0x79, 0x79, 0xbe, 0xb3, 0xfb, 0x7c, 0xe6, 0x79, 0x9e, 0x9d, 0x01, 0xb7, 0xbc, 0x7e, - 0xb7, 0xbe, 0x37, 0xc0, 0xbe, 0x89, 0x7d, 0xfe, 0x7f, 0xe0, 0x23, 0xa7, 0x8b, 0x13, 0xcd, 0x9a, - 0xe7, 0xbb, 0xd4, 0x85, 0x60, 0x3c, 0xb2, 0xbe, 0xd6, 0x75, 0xbb, 0x2e, 0x1f, 0xae, 0xb3, 0x96, - 0xb0, 0x58, 0xd7, 0xba, 0xae, 0xdb, 0xb5, 0x70, 0x9d, 0xf7, 0xda, 0x83, 0xdd, 0x3a, 0x35, 0x6d, - 0x4c, 0x28, 0xb2, 0xbd, 0xd0, 0xe0, 0x75, 0xf6, 0x2c, 0xcb, 0xed, 0x0a, 0x65, 0xd4, 0x08, 0x27, - 0x5f, 0x4b, 0x4d, 0x92, 0x3e, 0xa6, 0x3b, 0xbd, 0x70, 0xaa, 0x1c, 0x4e, 0xed, 0x59, 0xb6, 0xdb, - 0xc1, 0x56, 0x9d, 0x50, 0x44, 0x89, 0xf8, 0x0d, 0x2d, 0x56, 0x99, 0x85, 0x37, 0x20, 0x3d, 0xfe, - 0x13, 0x0e, 0x7e, 0xf6, 0x42, 0xd7, 0xda, 0x88, 0xe0, 0x7a, 0x07, 0xef, 0x9a, 0x8e, 0x49, 0x4d, - 0xd7, 0x21, 0xc9, 0x76, 0xb8, 0xc8, 0x9d, 0xf9, 0x16, 0x99, 0xc4, 0x55, 0x39, 0xca, 0x80, 0xd2, - 0x43, 0xb7, 0x6f, 0x1a, 0x78, 0x6f, 0x80, 0x09, 0x85, 0x6b, 0x20, 0xcf, 0x6d, 0x14, 0xb9, 0x2c, - 0x57, 0x8b, 0x86, 0xe8, 0xb0, 0x51, 0xcb, 0xb4, 0x4d, 0xaa, 0x64, 0xca, 0x72, 0xf5, 0xaa, 0x21, - 0x3a, 0x10, 0x82, 0x1c, 0xa1, 0xd8, 0x53, 0xb2, 0x65, 0xb9, 0x9a, 0x35, 0x78, 0x1b, 0xae, 0x83, - 0x65, 0xd3, 0xa1, 0xd8, 0xdf, 0x47, 0x96, 0x52, 0xe4, 0xe3, 0x71, 0x1f, 0x7e, 0x02, 0x96, 0x08, - 0x45, 0x3e, 0x6d, 0x11, 0x25, 0x57, 0x96, 0xab, 0xa5, 0xc6, 0x7a, 0x4d, 0x6c, 0x45, 0x2d, 0xda, - 0x8a, 0x5a, 0x2b, 0xda, 0x0a, 0x7d, 0xf9, 0x30, 0xd0, 0xa4, 0xa7, 0x7f, 0x68, 0xb2, 0x11, 0x89, - 0xe0, 0x07, 0x20, 0x8f, 0x9d, 0x4e, 0x8b, 0x28, 0xf9, 0x05, 0xd4, 0x42, 0x02, 0xdf, 0x05, 0xc5, - 0x8e, 0xe9, 0xe3, 0x1d, 0xc6, 0x4c, 0x29, 0x94, 0xe5, 0xea, 0x4a, 0x63, 0xb5, 0x16, 0x6f, 0xed, - 0x66, 0x34, 0x65, 0x8c, 0xad, 0x98, 0x7b, 0x1e, 0xa2, 0x3d, 0x65, 0x89, 0x93, 0xe0, 0x6d, 0x58, - 0x01, 0x05, 0xd2, 0x43, 0x7e, 0x87, 0x28, 0xcb, 0xe5, 0x6c, 0xb5, 0xa8, 0x83, 0xd3, 0x40, 0x0b, - 0x47, 0x8c, 0xf0, 0xbf, 0xf2, 0xb7, 0x0c, 0x20, 0x43, 0xba, 0xe5, 0x10, 0x8a, 0x1c, 0xfa, 0x32, - 0x64, 0x3f, 0x02, 0x05, 0x16, 0x94, 0x2d, 0xc2, 0xd9, 0xce, 0xeb, 0x6a, 0xa8, 0x49, 0xfb, 0x9a, - 0x5b, 0xc8, 0xd7, 0xfc, 0x4c, 0x5f, 0x0b, 0xe7, 0xfa, 0xfa, 0x73, 0x0e, 0x5c, 0x11, 0xe1, 0x43, - 0x3c, 0xd7, 0x21, 0x98, 0x89, 0xb6, 0x29, 0xa2, 0x03, 0x22, 0xdc, 0x0c, 0x45, 0x7c, 0xc4, 0x08, - 0x67, 0xe0, 0xa7, 0x20, 0xb7, 0x89, 0x28, 0xe2, 0x2e, 0x97, 0x1a, 0x6b, 0xb5, 0x44, 0x50, 0xb2, - 0xb5, 0xd8, 0x9c, 0x7e, 0x9d, 0x79, 0x75, 0x1a, 0x68, 0x2b, 0x1d, 0x44, 0xd1, 0xdb, 0xae, 0x6d, - 0x52, 0x6c, 0x7b, 0xf4, 0xc0, 0xe0, 0x4a, 0xf8, 0x1e, 0x28, 0x7e, 0xee, 0xfb, 0xae, 0xdf, 0x3a, - 0xf0, 0x30, 0x47, 0x54, 0xd4, 0x6f, 0x9c, 0x06, 0xda, 0x2a, 0x8e, 0x06, 0x13, 0x8a, 0xb1, 0x25, - 0x7c, 0x0b, 0xe4, 0x79, 0x87, 0x43, 0x29, 0xea, 0xab, 0xa7, 0x81, 0xf6, 0x3f, 0x2e, 0x49, 0x98, - 0x0b, 0x8b, 0x34, 0xc3, 0xfc, 0x5c, 0x0c, 0xe3, 0xad, 0x2c, 0x24, 0xb7, 0x52, 0x01, 0x4b, 0xfb, - 0xd8, 0x27, 0x6c, 0x99, 0x25, 0x3e, 0x1e, 0x75, 0xe1, 0x7d, 0x00, 0x18, 0x18, 0x93, 0x50, 0x73, - 0x87, 0xc5, 0x13, 0x83, 0x71, 0xb5, 0x26, 0xca, 0x85, 0x81, 0xc9, 0xc0, 0xa2, 0x3a, 0x0c, 0x29, - 0x24, 0x0c, 0x8d, 0x44, 0x1b, 0x3e, 0x93, 0xc1, 0x52, 0x13, 0xa3, 0x0e, 0xf6, 0x89, 0x52, 0x2c, - 0x67, 0xab, 0xa5, 0xc6, 0x1b, 0xb5, 0x64, 0x6d, 0x78, 0xe4, 0xbb, 0x36, 0xa6, 0x3d, 0x3c, 0x20, - 0xd1, 0x06, 0x09, 0x6b, 0xbd, 0x3f, 0x0c, 0xb4, 0x76, 0xd7, 0xa4, 0xbd, 0x41, 0xbb, 0xb6, 0xe3, - 0xda, 0xf5, 0xae, 0x8f, 0x76, 0x91, 0x83, 0xea, 0x96, 0xdb, 0x37, 0xeb, 0x0b, 0xd7, 0xa3, 0x73, - 0x9f, 0x73, 0x1a, 0x68, 0xf2, 0x3b, 0x46, 0xf4, 0x8a, 0x95, 0xdf, 0x65, 0xf0, 0x7f, 0xb6, 0xc3, - 0xdb, 0x6c, 0x6d, 0x92, 0x48, 0x0c, 0x1b, 0xd1, 0x9d, 0x9e, 0x22, 0xb3, 0x30, 0x33, 0x44, 0x27, - 0x59, 0x2c, 0x32, 0xff, 0xaa, 0x58, 0x64, 0x17, 0x2f, 0x16, 0x51, 0x36, 0xe4, 0x66, 0x66, 0x43, - 0xfe, 0xdc, 0x6c, 0xf8, 0x3e, 0x2b, 0x32, 0x3f, 0xf2, 0x6f, 0x81, 0x9c, 0x78, 0x10, 0xe7, 0x44, - 0x96, 0xbf, 0x6d, 0x1c, 0x6a, 0x62, 0xad, 0xad, 0x0e, 0x76, 0xa8, 0xb9, 0x6b, 0x62, 0xff, 0x05, - 0x99, 0x91, 0x08, 0xb7, 0x6c, 0x3a, 0xdc, 0x92, 0xb1, 0x92, 0x7b, 0xe5, 0x63, 0x65, 0x22, 0x3b, - 0xf2, 0x2f, 0x91, 0x1d, 0x95, 0xb3, 0x0c, 0xb8, 0xce, 0xb6, 0xe3, 0x21, 0x6a, 0x63, 0xeb, 0x2b, - 0x64, 0x2f, 0xb8, 0x25, 0x6f, 0x26, 0xb6, 0xa4, 0xa8, 0xc3, 0xff, 0x90, 0xcf, 0x81, 0xfc, 0x47, - 0x19, 0x2c, 0x47, 0x35, 0x1c, 0xd6, 0x00, 0x10, 0x32, 0x5e, 0xa6, 0x05, 0xe8, 0x15, 0x26, 0xf6, - 0xe3, 0x51, 0x23, 0x61, 0x01, 0xbf, 0x01, 0x05, 0xd1, 0x0b, 0xb3, 0xe0, 0x46, 0x22, 0x0b, 0xa8, - 0x8f, 0x91, 0x7d, 0xbf, 0x83, 0x3c, 0x8a, 0x7d, 0xfd, 0x1e, 0x7b, 0x8b, 0x61, 0xa0, 0xdd, 0xba, - 0x08, 0x11, 0x3f, 0x61, 0x09, 0x1d, 0xdb, 0x5c, 0xf1, 0x4c, 0x23, 0x7c, 0x42, 0xe5, 0x07, 0x19, - 0x5c, 0x63, 0x2f, 0xca, 0xd0, 0xc4, 0x51, 0xb1, 0x09, 0x96, 0xfd, 0xb0, 0xcd, 0x5f, 0xb7, 0xd4, - 0xa8, 0xd4, 0xd2, 0x58, 0x67, 0xa0, 0xd4, 0x73, 0x87, 0x81, 0x26, 0x1b, 0xb1, 0x12, 0x6e, 0xa4, - 0x30, 0x66, 0x66, 0x61, 0x64, 0x12, 0x29, 0x05, 0xee, 0xd7, 0x0c, 0x80, 0x5b, 0x4e, 0x07, 0x3f, - 0x61, 0xc1, 0x37, 0x8e, 0xd3, 0xc1, 0xd4, 0x1b, 0xdd, 0x1c, 0x43, 0x99, 0xb6, 0xd7, 0x3f, 0x1c, - 0x06, 0xda, 0xdd, 0x8b, 0xa8, 0x5c, 0x20, 0x4e, 0xb8, 0x90, 0x0c, 0xdc, 0xcc, 0xab, 0xff, 0x5d, - 0xf9, 0x25, 0x03, 0x56, 0xbe, 0x76, 0xad, 0x81, 0x8d, 0x63, 0x70, 0xf6, 0x14, 0x38, 0x65, 0x0c, - 0x2e, 0x6d, 0xab, 0xdf, 0x1d, 0x06, 0xda, 0xc6, 0x5c, 0xd0, 0xd2, 0xc2, 0xcb, 0x0b, 0xec, 0x59, - 0x06, 0xac, 0xb5, 0x5c, 0xef, 0x8b, 0x6d, 0x7e, 0x7d, 0x49, 0xd4, 0x45, 0x3c, 0x85, 0x6d, 0x6d, - 0x8c, 0x8d, 0x29, 0xbe, 0x44, 0xd4, 0x37, 0x9f, 0xe8, 0x1b, 0xc3, 0x40, 0xab, 0xcf, 0x85, 0x6c, - 0x2c, 0xba, 0xbc, 0xb8, 0x7e, 0xcb, 0x80, 0xeb, 0x8f, 0x07, 0xc8, 0xa1, 0xa6, 0x85, 0x05, 0xb2, - 0x18, 0xd8, 0xc1, 0x14, 0x30, 0x75, 0x0c, 0x2c, 0xad, 0x09, 0xd1, 0x7d, 0x3c, 0x0c, 0xb4, 0x7b, - 0x73, 0xa1, 0x9b, 0x25, 0xbf, 0xbc, 0x10, 0xbf, 0xcb, 0x81, 0xab, 0x8f, 0xd9, 0x2a, 0x31, 0xbb, - 0xf7, 0x41, 0x81, 0xf0, 0xd3, 0x4d, 0x4c, 0x6e, 0xe2, 0x26, 0x90, 0x3e, 0x47, 0x35, 0x25, 0x23, - 0xb4, 0x67, 0xf7, 0x23, 0x8b, 0x7d, 0xd4, 0xa3, 0xf2, 0x5a, 0x99, 0x54, 0x4e, 0x7f, 0xf2, 0x99, - 0x5a, 0x68, 0xe0, 0x1d, 0x90, 0xe7, 0xd5, 0x38, 0x3c, 0x1a, 0xa6, 0x1e, 0x3b, 0x5d, 0x16, 0x9b, - 0x92, 0x21, 0xcc, 0x61, 0x03, 0xe4, 0x3c, 0xdf, 0xb5, 0xc3, 0xcb, 0xeb, 0xcd, 0xc9, 0x67, 0x26, - 0x3f, 0x25, 0x4d, 0xc9, 0xe0, 0xb6, 0xf0, 0x36, 0x3b, 0xc6, 0xb2, 0x6f, 0x50, 0xf4, 0x41, 0x55, - 0x26, 0x65, 0x09, 0x49, 0x64, 0x0a, 0x6f, 0x83, 0xc2, 0x3e, 0x2f, 0x35, 0xfc, 0x2e, 0xc1, 0xce, - 0x83, 0x09, 0x51, 0xba, 0x08, 0x31, 0xbf, 0x84, 0x2d, 0x7c, 0x00, 0xae, 0x50, 0xd7, 0xeb, 0x47, - 0x49, 0xcd, 0xef, 0x1b, 0xa5, 0x46, 0x39, 0xa9, 0x9d, 0x95, 0xf4, 0x4d, 0xc9, 0x48, 0xe9, 0xe0, - 0x23, 0x70, 0x6d, 0x2f, 0x15, 0x7a, 0x38, 0xba, 0x9e, 0xa4, 0x38, 0xcf, 0xce, 0x88, 0xa6, 0x64, - 0x4c, 0xa9, 0x75, 0x30, 0xce, 0x12, 0xfd, 0xf6, 0xd1, 0xb1, 0x2a, 0x3d, 0x3f, 0x56, 0xa5, 0xb3, - 0x63, 0x55, 0xfe, 0x76, 0xa4, 0xca, 0x3f, 0x8d, 0x54, 0xf9, 0x70, 0xa4, 0xca, 0x47, 0x23, 0x55, - 0xfe, 0x73, 0xa4, 0xca, 0x7f, 0x8d, 0x54, 0xe9, 0x6c, 0xa4, 0xca, 0x4f, 0x4f, 0x54, 0xe9, 0xe8, - 0x44, 0x95, 0x9e, 0x9f, 0xa8, 0x52, 0xbb, 0xc0, 0x53, 0x62, 0xe3, 0x9f, 0x00, 0x00, 0x00, 0xff, - 0xff, 0x57, 0xdc, 0x66, 0x15, 0x08, 0x12, 0x00, 0x00, + // 1276 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x97, 0xcb, 0x8f, 0xdb, 0x44, + 0x1c, 0xc7, 0xed, 0xbc, 0x76, 0x33, 0xdb, 0x2e, 0x65, 0x76, 0xd5, 0x9a, 0xa5, 0xb2, 0xa3, 0x48, + 0xd0, 0x20, 0x81, 0x23, 0xb2, 0xa5, 0xa5, 0xbc, 0x44, 0x4d, 0xa9, 0xb6, 0xa2, 0xa0, 0xd6, 0x8d, + 0x38, 0x70, 0x9b, 0x24, 0xb3, 0x89, 0x89, 0x5f, 0xeb, 0x99, 0x54, 0xdd, 0x1b, 0x17, 0x6e, 0x20, + 0xf5, 0xaf, 0x40, 0x48, 0x54, 0xfc, 0x01, 0x1c, 0x39, 0xf5, 0xb8, 0xc7, 0x2a, 0x12, 0x86, 0xa6, + 0x17, 0xd8, 0x53, 0xff, 0x04, 0x34, 0x33, 0xb6, 0x33, 0x4e, 0xd2, 0x36, 0x29, 0x97, 0x56, 0xe2, + 0x92, 0xcc, 0xe3, 0xf7, 0x1d, 0xcf, 0x7c, 0x7e, 0x0f, 0x7b, 0xc0, 0xb9, 0x70, 0xd8, 0x6f, 0x1e, + 0x8c, 0x70, 0xe4, 0xe0, 0x88, 0xff, 0x1f, 0x46, 0xc8, 0xef, 0x63, 0xa9, 0x69, 0x86, 0x51, 0x40, + 0x03, 0x08, 0xa6, 0x23, 0x3b, 0xad, 0xbe, 0x43, 0x07, 0xa3, 0x8e, 0xd9, 0x0d, 0xbc, 0x66, 0x3f, + 0xe8, 0x07, 0xcd, 0x7e, 0x10, 0xf4, 0x5d, 0x8c, 0x42, 0x87, 0x24, 0xcd, 0x66, 0x14, 0x76, 0x9b, + 0x84, 0x22, 0x3a, 0x22, 0x42, 0xbf, 0xb3, 0xcd, 0x0c, 0x79, 0x93, 0x4b, 0x92, 0x51, 0x23, 0x31, + 0xe7, 0xbd, 0xce, 0x68, 0xbf, 0x49, 0x1d, 0x0f, 0x13, 0x8a, 0xbc, 0x30, 0x31, 0x78, 0x9d, 0xed, + 0xcf, 0x0d, 0xfa, 0x42, 0x99, 0x36, 0x92, 0xc9, 0xd7, 0x72, 0x93, 0x64, 0x88, 0x69, 0x77, 0x90, + 0x4c, 0xd5, 0x92, 0xa9, 0x03, 0xd7, 0x0b, 0x7a, 0xd8, 0xe5, 0x7b, 0x21, 0xe2, 0x37, 0xb1, 0xd8, + 0x62, 0x16, 0xe1, 0x88, 0x0c, 0xf8, 0x4f, 0x32, 0xf8, 0xd9, 0x33, 0x71, 0x74, 0x10, 0xc1, 0xcd, + 0x1e, 0xde, 0x77, 0x7c, 0x87, 0x3a, 0x81, 0x4f, 0xe4, 0x76, 0xb2, 0xc8, 0x85, 0xe5, 0x16, 0x99, + 0x45, 0x5c, 0x3f, 0x2a, 0x80, 0x8d, 0xeb, 0xc1, 0xd0, 0xb1, 0xf1, 0xc1, 0x08, 0x13, 0x0a, 0xb7, + 0x41, 0x99, 0xdb, 0x68, 0x6a, 0x4d, 0x6d, 0x54, 0x6d, 0xd1, 0x61, 0xa3, 0xae, 0xe3, 0x39, 0x54, + 0x2b, 0xd4, 0xd4, 0xc6, 0x49, 0x5b, 0x74, 0x20, 0x04, 0x25, 0x42, 0x71, 0xa8, 0x15, 0x6b, 0x6a, + 0xa3, 0x68, 0xf3, 0x36, 0xdc, 0x01, 0xeb, 0x8e, 0x4f, 0x71, 0x74, 0x1b, 0xb9, 0x5a, 0x95, 0x8f, + 0x67, 0x7d, 0xf8, 0x09, 0x58, 0x23, 0x14, 0x45, 0xb4, 0x4d, 0xb4, 0x52, 0x4d, 0x6d, 0x6c, 0xb4, + 0x76, 0x4c, 0xe1, 0x0a, 0x33, 0x75, 0x85, 0xd9, 0x4e, 0x5d, 0x61, 0xad, 0xdf, 0x8f, 0x0d, 0xe5, + 0xee, 0x9f, 0x86, 0x6a, 0xa7, 0x22, 0xf8, 0x01, 0x28, 0x63, 0xbf, 0xd7, 0x26, 0x5a, 0x79, 0x05, + 0xb5, 0x90, 0xc0, 0x77, 0x41, 0xb5, 0xe7, 0x44, 0xb8, 0xcb, 0x98, 0x69, 0x95, 0x9a, 0xda, 0xd8, + 0x6c, 0x6d, 0x99, 0x99, 0x6b, 0xaf, 0xa4, 0x53, 0xf6, 0xd4, 0x8a, 0x1d, 0x2f, 0x44, 0x74, 0xa0, + 0xad, 0x71, 0x12, 0xbc, 0x0d, 0xeb, 0xa0, 0x42, 0x06, 0x28, 0xea, 0x11, 0x6d, 0xbd, 0x56, 0x6c, + 0x54, 0x2d, 0x70, 0x1c, 0x1b, 0xc9, 0x88, 0x9d, 0xfc, 0xd7, 0xff, 0x51, 0x01, 0x64, 0x48, 0xaf, + 0xf9, 0x84, 0x22, 0x9f, 0x3e, 0x0f, 0xd9, 0x8f, 0x40, 0x85, 0x05, 0x65, 0x9b, 0x70, 0xb6, 0xcb, + 0x1e, 0x35, 0xd1, 0xe4, 0xcf, 0x5a, 0x5a, 0xe9, 0xac, 0xe5, 0x85, 0x67, 0xad, 0x3c, 0xf1, 0xac, + 0xbf, 0x94, 0xc0, 0x09, 0x11, 0x3e, 0x24, 0x0c, 0x7c, 0x82, 0x99, 0xe8, 0x16, 0x4f, 0x41, 0x71, + 0xcc, 0x44, 0xc4, 0x47, 0xec, 0x64, 0x06, 0x7e, 0x0a, 0x4a, 0x57, 0x10, 0x45, 0xfc, 0xc8, 0x1b, + 0xad, 0x6d, 0x53, 0x0a, 0x4a, 0xb6, 0x16, 0x9b, 0xb3, 0x4e, 0xb3, 0x53, 0x1d, 0xc7, 0xc6, 0x66, + 0x0f, 0x51, 0xf4, 0x76, 0xe0, 0x39, 0x14, 0x7b, 0x21, 0x3d, 0xb4, 0xb9, 0x12, 0xbe, 0x07, 0xaa, + 0x9f, 0x47, 0x51, 0x10, 0xb5, 0x0f, 0x43, 0xcc, 0x11, 0x55, 0xad, 0x33, 0xc7, 0xb1, 0xb1, 0x85, + 0xd3, 0x41, 0x49, 0x31, 0xb5, 0x84, 0x6f, 0x81, 0x32, 0xef, 0x70, 0x28, 0x55, 0x6b, 0xeb, 0x38, + 0x36, 0x5e, 0xe1, 0x12, 0xc9, 0x5c, 0x58, 0xe4, 0x19, 0x96, 0x97, 0x62, 0x98, 0xb9, 0xb2, 0x22, + 0xbb, 0x52, 0x03, 0x6b, 0xb7, 0x71, 0x44, 0xd8, 0x32, 0x6b, 0x7c, 0x3c, 0xed, 0xc2, 0xcb, 0x00, + 0x30, 0x30, 0x0e, 0xa1, 0x4e, 0x97, 0xc5, 0x13, 0x83, 0x71, 0xd2, 0x14, 0xe5, 0xc2, 0xc6, 0x64, + 0xe4, 0x52, 0x0b, 0x26, 0x14, 0x24, 0x43, 0x5b, 0x6a, 0xc3, 0x7b, 0x2a, 0x58, 0xdb, 0xc3, 0xa8, + 0x87, 0x23, 0xa2, 0x55, 0x6b, 0xc5, 0xc6, 0x46, 0xeb, 0x0d, 0x53, 0xae, 0x0d, 0x37, 0xa2, 0xc0, + 0xc3, 0x74, 0x80, 0x47, 0x24, 0x75, 0x90, 0xb0, 0xb6, 0x86, 0xe3, 0xd8, 0xe8, 0xc8, 0x15, 0x35, + 0x42, 0xfb, 0xc8, 0x47, 0x4d, 0x37, 0x18, 0x3a, 0xcd, 0x95, 0xeb, 0xd1, 0x13, 0x9f, 0x73, 0x1c, + 0x1b, 0xea, 0x3b, 0x76, 0xba, 0xc5, 0xfa, 0x1f, 0x2a, 0x78, 0x95, 0x79, 0xf8, 0x16, 0x5b, 0x9b, + 0x48, 0x89, 0xe1, 0x21, 0xda, 0x1d, 0x68, 0x2a, 0x0b, 0x33, 0x5b, 0x74, 0xe4, 0x62, 0x51, 0xf8, + 0x4f, 0xc5, 0xa2, 0xb8, 0x7a, 0xb1, 0x48, 0xb3, 0xa1, 0xb4, 0x30, 0x1b, 0xca, 0x4f, 0xcc, 0x86, + 0x1f, 0x8a, 0x22, 0xf3, 0xd3, 0xf3, 0xad, 0x90, 0x13, 0x57, 0xb3, 0x9c, 0x28, 0xf2, 0xdd, 0x66, + 0xa1, 0x26, 0xd6, 0xba, 0xd6, 0xc3, 0x3e, 0x75, 0xf6, 0x1d, 0x1c, 0x3d, 0x23, 0x33, 0xa4, 0x70, + 0x2b, 0xe6, 0xc3, 0x4d, 0x8e, 0x95, 0xd2, 0x0b, 0x1f, 0x2b, 0x33, 0xd9, 0x51, 0x7e, 0x8e, 0xec, + 0xa8, 0x3f, 0x2e, 0x80, 0xd3, 0xcc, 0x1d, 0xd7, 0x51, 0x07, 0xbb, 0x5f, 0x21, 0x6f, 0x45, 0x97, + 0xbc, 0x29, 0xb9, 0xa4, 0x6a, 0xc1, 0xff, 0x91, 0x2f, 0x81, 0xfc, 0x27, 0x15, 0xac, 0xa7, 0x35, + 0x1c, 0x9a, 0x00, 0x08, 0x19, 0x2f, 0xd3, 0x02, 0xf4, 0x26, 0x13, 0x47, 0xd9, 0xa8, 0x2d, 0x59, + 0xc0, 0x6f, 0x41, 0x45, 0xf4, 0x92, 0x2c, 0x38, 0x23, 0x65, 0x01, 0x8d, 0x30, 0xf2, 0x2e, 0xf7, + 0x50, 0x48, 0x71, 0x64, 0x5d, 0x62, 0xbb, 0x18, 0xc7, 0xc6, 0xb9, 0xa7, 0x21, 0xe2, 0x5f, 0x58, + 0x42, 0xc7, 0x9c, 0x2b, 0x9e, 0x69, 0x27, 0x4f, 0xa8, 0xff, 0xa8, 0x82, 0x53, 0x6c, 0xa3, 0x0c, + 0x4d, 0x16, 0x15, 0x57, 0xc0, 0x7a, 0x94, 0xb4, 0xf9, 0x76, 0x37, 0x5a, 0x75, 0x33, 0x8f, 0x75, + 0x01, 0x4a, 0xab, 0x74, 0x3f, 0x36, 0x54, 0x3b, 0x53, 0xc2, 0xdd, 0x1c, 0xc6, 0xc2, 0x22, 0x8c, + 0x4c, 0xa2, 0xe4, 0xc0, 0xfd, 0x56, 0x00, 0xf0, 0x9a, 0xdf, 0xc3, 0x77, 0x58, 0xf0, 0x4d, 0xe3, + 0x74, 0x34, 0xb7, 0xa3, 0xb3, 0x53, 0x28, 0xf3, 0xf6, 0xd6, 0x87, 0xe3, 0xd8, 0xb8, 0xf8, 0x34, + 0x2a, 0x4f, 0x11, 0x4b, 0x47, 0x90, 0x03, 0xb7, 0xf0, 0xe2, 0xbf, 0x57, 0x7e, 0x2d, 0x80, 0xcd, + 0xaf, 0x03, 0x77, 0xe4, 0xe1, 0x0c, 0x9c, 0x37, 0x07, 0x4e, 0x9b, 0x82, 0xcb, 0xdb, 0x5a, 0x17, + 0xc7, 0xb1, 0xb1, 0xbb, 0x14, 0xb4, 0xbc, 0xf0, 0xe5, 0x05, 0x76, 0xaf, 0x00, 0xb6, 0xdb, 0x41, + 0xf8, 0xc5, 0x2d, 0x7e, 0x7d, 0x91, 0xea, 0x22, 0x9e, 0xc3, 0xb6, 0x3d, 0xc5, 0xc6, 0x14, 0x5f, + 0x22, 0x1a, 0x39, 0x77, 0xac, 0xdd, 0x71, 0x6c, 0x34, 0x97, 0x42, 0x36, 0x15, 0xbd, 0xbc, 0xb8, + 0x7e, 0x2f, 0x80, 0xd3, 0x37, 0x47, 0xc8, 0xa7, 0x8e, 0x8b, 0x05, 0xb2, 0x0c, 0xd8, 0xe1, 0x1c, + 0x30, 0x7d, 0x0a, 0x2c, 0xaf, 0x49, 0xd0, 0x7d, 0x3c, 0x8e, 0x8d, 0x4b, 0x4b, 0xa1, 0x5b, 0x24, + 0x7f, 0x79, 0x21, 0x7e, 0x5f, 0x02, 0x27, 0x6f, 0xb2, 0x55, 0x32, 0x76, 0xef, 0x83, 0x0a, 0xe1, + 0x5f, 0x37, 0x19, 0xb9, 0x99, 0x9b, 0x40, 0xfe, 0x3b, 0x6a, 0x4f, 0xb1, 0x13, 0x7b, 0x76, 0x3f, + 0x72, 0xd9, 0x4b, 0x3d, 0x2d, 0xaf, 0xf5, 0x59, 0xe5, 0xfc, 0x2b, 0x9f, 0xa9, 0x85, 0x06, 0x5e, + 0x00, 0x65, 0x5e, 0x8d, 0x93, 0x4f, 0xc3, 0xdc, 0x63, 0xe7, 0xcb, 0xe2, 0x9e, 0x62, 0x0b, 0x73, + 0xd8, 0x02, 0xa5, 0x30, 0x0a, 0xbc, 0xe4, 0xf2, 0x7a, 0x76, 0xf6, 0x99, 0xf2, 0xab, 0x64, 0x4f, + 0xb1, 0xb9, 0x2d, 0x3c, 0xcf, 0x3e, 0x63, 0xd9, 0x3b, 0x28, 0x7d, 0xa1, 0x6a, 0xb3, 0x32, 0x49, + 0x92, 0x9a, 0xc2, 0xf3, 0xa0, 0x72, 0x9b, 0x97, 0x1a, 0x7e, 0x97, 0x60, 0xdf, 0x83, 0x92, 0x28, + 0x5f, 0x84, 0xd8, 0xb9, 0x84, 0x2d, 0xbc, 0x0a, 0x4e, 0xd0, 0x20, 0x1c, 0xa6, 0x49, 0xcd, 0xef, + 0x1b, 0x1b, 0xad, 0x9a, 0xac, 0x5d, 0x94, 0xf4, 0x7b, 0x8a, 0x9d, 0xd3, 0xc1, 0x1b, 0xe0, 0xd4, + 0x41, 0x2e, 0xf4, 0x70, 0x7a, 0x3d, 0xc9, 0x71, 0x5e, 0x9c, 0x11, 0x7b, 0x8a, 0x3d, 0xa7, 0xb6, + 0xc0, 0x34, 0x4b, 0xac, 0xde, 0xd1, 0x43, 0x5d, 0x79, 0xf0, 0x50, 0x57, 0x1e, 0x3f, 0xd4, 0xd5, + 0xef, 0x26, 0xba, 0xfa, 0xf3, 0x44, 0x57, 0xef, 0x4f, 0x74, 0xf5, 0x68, 0xa2, 0xab, 0x7f, 0x4d, + 0x74, 0xf5, 0xef, 0x89, 0xae, 0x3c, 0x9e, 0xe8, 0xea, 0xdd, 0x47, 0xba, 0x72, 0xf4, 0x48, 0x57, + 0x1e, 0x3c, 0xd2, 0x95, 0x6f, 0xcc, 0xd5, 0xe2, 0xb5, 0x53, 0xe1, 0x29, 0xb4, 0xfb, 0x6f, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xe2, 0x6b, 0x8d, 0x42, 0x6c, 0x12, 0x00, 0x00, } func (this *LokiRequest) Equal(that interface{}) bool { diff --git a/pkg/querier/queryrange/queryrange.proto b/pkg/querier/queryrange/queryrange.proto index 0a02a5c615..c299b7275f 100644 --- a/pkg/querier/queryrange/queryrange.proto +++ b/pkg/querier/queryrange/queryrange.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package queryrange; +import "github.com/gogo/googleapis/google/rpc/status.proto"; import "gogoproto/gogo.proto"; import "google/protobuf/timestamp.proto"; import "pkg/logproto/logproto.proto"; @@ -11,6 +12,7 @@ import "pkg/push/push.proto"; import "pkg/querier/queryrange/queryrangebase/definitions/definitions.proto"; import "pkg/querier/queryrange/queryrangebase/queryrange.proto"; +option go_package = "github.com/grafana/loki/pkg/querier/queryrange"; option (gogoproto.marshaler_all) = true; option (gogoproto.unmarshaler_all) = true; diff --git a/pkg/querier/queryrange/querysharding.go b/pkg/querier/queryrange/querysharding.go index 2babe0e7df..038e0611f9 100644 --- a/pkg/querier/queryrange/querysharding.go +++ b/pkg/querier/queryrange/querysharding.go @@ -204,7 +204,7 @@ func (ast *astMapperware) Do(ctx context.Context, r queryrangebase.Request) (que return ast.next.Do(ctx, r) } - params, err := paramsFromRequest(r) + params, err := ParamsFromRequest(r) if err != nil { return nil, err } diff --git a/pkg/querier/queryrange/roundtrip_test.go b/pkg/querier/queryrange/roundtrip_test.go index 0ef20055ef..a06dc98e93 100644 --- a/pkg/querier/queryrange/roundtrip_test.go +++ b/pkg/querier/queryrange/roundtrip_test.go @@ -487,10 +487,10 @@ func TestLabelsTripperware(t *testing.T) { handler := newFakeHandler( // we expect 2 calls. http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, marshal.WriteLabelResponseJSON(logproto.LabelResponse{Values: []string{"foo", "bar", "blop"}}, w)) + require.NoError(t, marshal.WriteLabelResponseJSON([]string{"foo", "bar", "blop"}, w)) }), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, marshal.WriteLabelResponseJSON(logproto.LabelResponse{Values: []string{"foo", "bar", "blip"}}, w)) + require.NoError(t, marshal.WriteLabelResponseJSON([]string{"foo", "bar", "blip"}, w)) }), ) rt.setHandler(handler) @@ -1490,7 +1490,7 @@ func promqlResult(v parser.Value) (*int, http.Handler) { return &count, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { lock.Lock() defer lock.Unlock() - if err := marshal.WriteQueryResponseJSON(logqlmodel.Result{Data: v}, w); err != nil { + if err := marshal.WriteQueryResponseJSON(v, stats.Result{}, w); err != nil { panic(err) } count++ @@ -1503,7 +1503,7 @@ func seriesResult(v logproto.SeriesResponse) (*int, http.Handler) { return &count, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { lock.Lock() defer lock.Unlock() - if err := marshal.WriteSeriesResponseJSON(v, w); err != nil { + if err := marshal.WriteSeriesResponseJSON(v.GetSeries(), w); err != nil { panic(err) } count++ diff --git a/pkg/querier/queryrange/serialize.go b/pkg/querier/queryrange/serialize.go new file mode 100644 index 0000000000..df9e49af37 --- /dev/null +++ b/pkg/querier/queryrange/serialize.go @@ -0,0 +1,76 @@ +package queryrange + +import ( + "net/http" + + "github.com/opentracing/opentracing-go" + + "github.com/grafana/loki/pkg/loghttp" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" + serverutil "github.com/grafana/loki/pkg/util/server" +) + +type serializeRoundTripper struct { + codec queryrangebase.Codec + next queryrangebase.Handler +} + +func NewSerializeRoundTripper(next queryrangebase.Handler, codec queryrangebase.Codec) http.RoundTripper { + return &serializeRoundTripper{ + next: next, + codec: codec, + } +} + +func (rt *serializeRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + ctx := r.Context() + sp, ctx := opentracing.StartSpanFromContext(ctx, "limitedRoundTripper.do") + defer sp.Finish() + + request, err := rt.codec.DecodeRequest(ctx, r, nil) + if err != nil { + return nil, err + } + + response, err := rt.next.Do(ctx, request) + if err != nil { + return nil, err + } + + return rt.codec.EncodeResponse(ctx, r, response) +} + +type serializeHTTPHandler struct { + codec queryrangebase.Codec + next queryrangebase.Handler +} + +func NewSerializeHTTPHandler(next queryrangebase.Handler, codec queryrangebase.Codec) http.Handler { + return &serializeHTTPHandler{ + next: next, + codec: codec, + } +} + +func (rt *serializeHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + sp, ctx := opentracing.StartSpanFromContext(ctx, "serializeHTTPHandler.ServerHTTP") + defer sp.Finish() + + request, err := rt.codec.DecodeRequest(ctx, r, nil) + if err != nil { + serverutil.WriteError(err, w) + return + } + + response, err := rt.next.Do(ctx, request) + if err != nil { + serverutil.WriteError(err, w) + return + } + + version := loghttp.GetVersion(r.RequestURI) + if err := encodeResponseJSONTo(version, response, w); err != nil { + serverutil.WriteError(err, w) + } +} diff --git a/pkg/querier/queryrange/serialize_test.go b/pkg/querier/queryrange/serialize_test.go new file mode 100644 index 0000000000..6f276f39b2 --- /dev/null +++ b/pkg/querier/queryrange/serialize_test.go @@ -0,0 +1,132 @@ +package queryrange + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/grafana/dskit/user" + "github.com/stretchr/testify/require" + + "github.com/grafana/loki/pkg/loghttp" + "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" +) + +func TestResponseFormat(t *testing.T) { + for _, tc := range []struct { + url string + accept string + response queryrangebase.Response + expectedCode int + expectedRespone string + }{ + { + url: "/api/prom/query", + response: &LokiResponse{ + Direction: logproto.BACKWARD, + Limit: 200, + Data: LokiData{ + ResultType: loghttp.ResultTypeStream, + Result: logqlmodel.Streams{ + logproto.Stream{ + Entries: []logproto.Entry{ + { + Timestamp: time.Unix(0, 123456789012345).UTC(), + Line: "super line", + }, + }, + Labels: `{foo="bar"}`, + }, + }, + }, + Status: "success", + Statistics: statsResult, + }, + expectedCode: http.StatusOK, + expectedRespone: `{ + ` + statsResultString + ` + "streams": [ + { + "labels": "{foo=\"bar\"}", + "entries": [ + { + "line": "super line", + "ts": "1970-01-02T10:17:36.789012345Z" + } + ] + } + ] + }`, + }, + { + url: "/loki/api/v1/query_range", + response: &LokiResponse{ + Direction: logproto.BACKWARD, + Limit: 200, + Data: LokiData{ + ResultType: loghttp.ResultTypeStream, + Result: logqlmodel.Streams{ + logproto.Stream{ + Entries: []logproto.Entry{ + { + Timestamp: time.Unix(0, 123456789012345).UTC(), + Line: "super line", + }, + }, + Labels: `{foo="bar"}`, + }, + }, + }, + Status: "success", + Statistics: statsResult, + }, + expectedCode: http.StatusOK, + expectedRespone: `{ + "status": "success", + "data": { + "resultType": "streams", + ` + statsResultString + ` + "result": [{ + "stream": {"foo": "bar"}, + "values": [ + ["123456789012345", "super line"] + ] + }] + } + }`, + }, + { + url: "/loki/wrong/path", + response: nil, + expectedCode: http.StatusNotFound, + expectedRespone: "unknown request path: /loki/wrong/path\n", + }, + } { + t.Run(fmt.Sprintf("%s returns the expected format", tc.url), func(t *testing.T) { + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + return tc.response, nil + }) + httpHandler := NewSerializeHTTPHandler(handler, DefaultCodec) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, tc.url+ + "?start=0"+ + "&end=1"+ + "&query=%7Bfoo%3D%22bar%22%7D", nil) + req = req.WithContext(user.InjectOrgID(context.Background(), "1")) + httpHandler.ServeHTTP(w, req) + + require.Equalf(t, tc.expectedCode, w.Code, "unexpected response: %s", w.Body.String()) + if tc.expectedCode/100 == 2 { + require.JSONEq(t, tc.expectedRespone, w.Body.String()) + } else { + require.Equal(t, tc.expectedRespone, w.Body.String()) + } + }) + } +} diff --git a/pkg/querier/queryrange/split_by_range.go b/pkg/querier/queryrange/split_by_range.go index 8fa24eb3b2..e3640761d5 100644 --- a/pkg/querier/queryrange/split_by_range.go +++ b/pkg/querier/queryrange/split_by_range.go @@ -80,7 +80,7 @@ func (s *splitByRange) Do(ctx context.Context, request queryrangebase.Request) ( queryStatsCtx := stats.FromContext(ctx) queryStatsCtx.AddSplitQueries(int64(mapperStats.GetSplitQueries())) - params, err := paramsFromRequest(request) + params, err := ParamsFromRequest(request) if err != nil { return nil, err } diff --git a/pkg/querier/queryrange/stats.go b/pkg/querier/queryrange/stats.go index 223d1c4271..7e0dc49fe5 100644 --- a/pkg/querier/queryrange/stats.go +++ b/pkg/querier/queryrange/stats.go @@ -173,7 +173,7 @@ func StatsCollectorMiddleware() queryrangebase.Middleware { data.statistics = responseStats data.result = res data.queryType = queryType - p, errReq := paramsFromRequest(req) + p, errReq := ParamsFromRequest(req) if errReq != nil { return nil, errReq } diff --git a/pkg/querier/queryrange/stats_test.go b/pkg/querier/queryrange/stats_test.go index c82e121a72..54c9004d88 100644 --- a/pkg/querier/queryrange/stats_test.go +++ b/pkg/querier/queryrange/stats_test.go @@ -99,7 +99,7 @@ func Test_StatsHTTP(t *testing.T) { http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { data := r.Context().Value(ctxKey).(*queryData) data.recorded = true - data.params, _ = paramsFromRequest(&LokiRequest{ + data.params, _ = ParamsFromRequest(&LokiRequest{ Query: "foo", Direction: logproto.BACKWARD, Limit: 100, @@ -119,7 +119,7 @@ func Test_StatsHTTP(t *testing.T) { http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { data := r.Context().Value(ctxKey).(*queryData) data.recorded = true - data.params, _ = paramsFromRequest(&LokiRequest{ + data.params, _ = ParamsFromRequest(&LokiRequest{ Query: "foo", Direction: logproto.BACKWARD, Limit: 100, @@ -140,7 +140,7 @@ func Test_StatsHTTP(t *testing.T) { http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { data := r.Context().Value(ctxKey).(*queryData) data.recorded = true - data.params, _ = paramsFromRequest(&LokiRequest{ + data.params, _ = ParamsFromRequest(&LokiRequest{ Query: "foo", Direction: logproto.BACKWARD, Limit: 100, @@ -163,7 +163,7 @@ func Test_StatsHTTP(t *testing.T) { http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { data := r.Context().Value(ctxKey).(*queryData) data.recorded = true - data.params, _ = paramsFromRequest(&logproto.VolumeRequest{ + data.params, _ = ParamsFromRequest(&logproto.VolumeRequest{ Matchers: "foo", Limit: 100, }) diff --git a/pkg/querier/queryrange/views_test.go b/pkg/querier/queryrange/views_test.go index eacf157074..adcd9d41ff 100644 --- a/pkg/querier/queryrange/views_test.go +++ b/pkg/querier/queryrange/views_test.go @@ -245,10 +245,7 @@ func TestMergedViewJSON(t *testing.T) { actual := b.String() b.Reset() - result := logproto.SeriesResponse{ - Series: response.Data, - } - err = marshal.WriteSeriesResponseJSON(result, &b) + err = marshal.WriteSeriesResponseJSON(response.Data, &b) require.NoError(t, err) expected := b.String() diff --git a/pkg/querier/worker/frontend_processor.go b/pkg/querier/worker/frontend_processor.go index 4394495db2..5a07944d79 100644 --- a/pkg/querier/worker/frontend_processor.go +++ b/pkg/querier/worker/frontend_processor.go @@ -10,10 +10,12 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/dskit/backoff" "github.com/grafana/dskit/httpgrpc" + "github.com/opentracing/opentracing-go" "google.golang.org/grpc" "github.com/grafana/loki/pkg/lokifrontend/frontend/v1/frontendv1pb" querier_stats "github.com/grafana/loki/pkg/querier/stats" + httpgrpcutil "github.com/grafana/loki/pkg/util/httpgrpc" ) var ( @@ -23,18 +25,21 @@ var ( } ) -func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger) processor { +func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger, codec GRPCCodec) processor { return &frontendProcessor{ log: log, handler: handler, + codec: codec, maxMessageSize: cfg.GRPCClientConfig.MaxSendMsgSize, querierID: cfg.QuerierID, } } -// Handles incoming queries from frontend. +// Handles incoming queries from frontend. This is used if there's no query-scheduler between the frontend and querier. +// This should be used by Frontend V1. type frontendProcessor struct { handler RequestHandler + codec GRPCCodec maxMessageSize int querierID string @@ -113,23 +118,24 @@ func (fp *frontendProcessor) process(c frontendv1pb.Frontend_ProcessClient) erro } } -func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, statsEnabled bool, sendHTTPResponse func(response *httpgrpc.HTTPResponse, stats *querier_stats.Stats) error) { +func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, statsEnabled bool, sendResponse func(response *httpgrpc.HTTPResponse, stats *querier_stats.Stats) error) { + + tracer := opentracing.GlobalTracer() + // Ignore errors here. If we cannot get parent span, we just don't create new one. + parentSpanContext, _ := httpgrpcutil.GetParentSpanForRequest(tracer, request) + if parentSpanContext != nil { + queueSpan, spanCtx := opentracing.StartSpanFromContextWithTracer(ctx, tracer, "frontend_processor_runRequest", opentracing.ChildOf(parentSpanContext)) + defer queueSpan.Finish() + + ctx = spanCtx + } + var stats *querier_stats.Stats if statsEnabled { stats, ctx = querier_stats.ContextWithEmptyStats(ctx) } - response, err := fp.handler.Handle(ctx, request) - if err != nil { - var ok bool - response, ok = httpgrpc.HTTPResponseFromError(err) - if !ok { - response = &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(err.Error()), - } - } - } + response := handle(ctx, request, fp.handler, fp.codec) // Ensure responses that are too big are not retried. if len(response.Body) >= fp.maxMessageSize { @@ -141,7 +147,7 @@ func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.H level.Error(fp.log).Log("msg", "error processing query", "err", errMsg) } - if err := sendHTTPResponse(response, stats); err != nil { + if err := sendResponse(response, stats); err != nil { level.Error(fp.log).Log("msg", "error processing requests", "err", err) } } diff --git a/pkg/querier/worker/frontend_processor_test.go b/pkg/querier/worker/frontend_processor_test.go index 9303599088..e446500dd8 100644 --- a/pkg/querier/worker/frontend_processor_test.go +++ b/pkg/querier/worker/frontend_processor_test.go @@ -14,6 +14,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/test/bufconn" + "github.com/grafana/loki/pkg/querier/queryrange" "github.com/grafana/loki/pkg/util/test" ) @@ -32,7 +33,7 @@ func TestRecvFailDoesntCancelProcess(t *testing.T) { require.NoError(t, err) cfg := Config{} - mgr := newFrontendProcessor(cfg, nil, log.NewNopLogger()) + mgr := newFrontendProcessor(cfg, nil, log.NewNopLogger(), queryrange.DefaultCodec) running := atomic.NewBool(false) go func() { running.Store(true) diff --git a/pkg/querier/worker/scheduler_processor.go b/pkg/querier/worker/scheduler_processor.go index 7465b772d6..b5ad89b755 100644 --- a/pkg/querier/worker/scheduler_processor.go +++ b/pkg/querier/worker/scheduler_processor.go @@ -31,10 +31,11 @@ import ( util_log "github.com/grafana/loki/pkg/util/log" ) -func newSchedulerProcessor(cfg Config, handler RequestHandler, log log.Logger, metrics *Metrics) (*schedulerProcessor, []services.Service) { +func newSchedulerProcessor(cfg Config, handler RequestHandler, log log.Logger, metrics *Metrics, codec GRPCCodec) (*schedulerProcessor, []services.Service) { p := &schedulerProcessor{ log: log, handler: handler, + codec: codec, maxMessageSize: cfg.GRPCClientConfig.MaxSendMsgSize, querierID: cfg.QuerierID, grpcConfig: cfg.GRPCClientConfig, @@ -58,6 +59,7 @@ func newSchedulerProcessor(cfg Config, handler RequestHandler, log log.Logger, m type schedulerProcessor struct { log log.Logger handler RequestHandler + codec GRPCCodec grpcConfig grpcclient.Config maxMessageSize int querierID string @@ -167,17 +169,7 @@ func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger, stats, ctx = querier_stats.ContextWithEmptyStats(ctx) } - response, err := sp.handler.Handle(ctx, request) - if err != nil { - var ok bool - response, ok = httpgrpc.HTTPResponseFromError(err) - if !ok { - response = &httpgrpc.HTTPResponse{ - Code: http.StatusInternalServerError, - Body: []byte(err.Error()), - } - } - } + response := handle(ctx, request, sp.handler, sp.codec) logger = log.With(logger, "frontend", frontendAddress) @@ -199,7 +191,7 @@ func (sp *schedulerProcessor) runRequest(ctx context.Context, logger log.Logger, frontendAddress, func(c client.PoolClient) error { // Response is empty and uninteresting. - _, err = c.(frontendv2pb.FrontendForQuerierClient).QueryResult(ctx, &frontendv2pb.QueryResultRequest{ + _, err := c.(frontendv2pb.FrontendForQuerierClient).QueryResult(ctx, &frontendv2pb.QueryResultRequest{ QueryID: queryID, HttpResponse: response, Stats: stats, diff --git a/pkg/querier/worker/scheduler_processor_test.go b/pkg/querier/worker/scheduler_processor_test.go index c720c93189..5056c1e030 100644 --- a/pkg/querier/worker/scheduler_processor_test.go +++ b/pkg/querier/worker/scheduler_processor_test.go @@ -19,6 +19,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "github.com/grafana/loki/pkg/querier/queryrange" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/scheduler/schedulerpb" ) @@ -37,7 +39,7 @@ func TestSchedulerProcessor_processQueriesOnSingleStream(t *testing.T) { return nil, loopClient.Context().Err() }) - requestHandler.On("Handle", mock.Anything, mock.Anything).Return(&httpgrpc.HTTPResponse{}, nil) + requestHandler.On("Do", mock.Anything, mock.Anything).Return(&queryrange.LokiResponse{}, nil) sp.processQueriesOnSingleStream(workerCtx, nil, "127.0.0.1") @@ -58,8 +60,11 @@ func TestSchedulerProcessor_processQueriesOnSingleStream(t *testing.T) { switch recvCount.Inc() { case 1: return &schedulerpb.SchedulerToQuerier{ - QueryID: 1, - HttpRequest: nil, + QueryID: 1, + HttpRequest: &httpgrpc.HTTPRequest{ + Method: "GET", + Url: `/loki/api/v1/query_range?query={foo="bar"}&step=10&limit=200&direction=FORWARD`, + }, FrontendAddress: "127.0.0.2", UserID: "user-1", }, nil @@ -72,7 +77,7 @@ func TestSchedulerProcessor_processQueriesOnSingleStream(t *testing.T) { workerCtx, workerCancel := context.WithCancel(context.Background()) - requestHandler.On("Handle", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + requestHandler.On("Do", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { // Cancel the worker context while the query execution is in progress. workerCancel() @@ -81,7 +86,7 @@ func TestSchedulerProcessor_processQueriesOnSingleStream(t *testing.T) { // Intentionally slow down the query execution, to double check the worker waits until done. time.Sleep(time.Second) - }).Return(&httpgrpc.HTTPResponse{}, nil) + }).Return(&queryrange.LokiResponse{}, nil) startTime := time.Now() sp.processQueriesOnSingleStream(workerCtx, nil, "127.0.0.1") @@ -113,7 +118,7 @@ func TestSchedulerProcessor_processQueriesOnSingleStream(t *testing.T) { return nil, status.Error(codes.Unknown, schedulerpb.ErrSchedulerIsNotRunning.Error()) }) - requestHandler.On("Handle", mock.Anything, mock.Anything).Return(&httpgrpc.HTTPResponse{}, nil) + requestHandler.On("Do", mock.Anything, mock.Anything).Return(&queryrange.LokiResponse{}, nil) sp.processQueriesOnSingleStream(workerCtx, nil, "127.0.0.1") @@ -139,7 +144,7 @@ func prepareSchedulerProcessor() (*schedulerProcessor, *querierLoopClientMock, * requestHandler := &requestHandlerMock{} metrics := NewMetrics(Config{}, nil) - sp, _ := newSchedulerProcessor(Config{QuerierID: "test-querier-id"}, requestHandler, log.NewNopLogger(), metrics) + sp, _ := newSchedulerProcessor(Config{QuerierID: "test-querier-id"}, requestHandler, log.NewNopLogger(), metrics, queryrange.DefaultCodec) sp.schedulerClientFactory = func(_ *grpc.ClientConn) schedulerpb.SchedulerForQuerierClient { return schedulerClient } @@ -221,7 +226,7 @@ type requestHandlerMock struct { mock.Mock } -func (m *requestHandlerMock) Handle(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) { +func (m *requestHandlerMock) Do(ctx context.Context, req queryrangebase.Request) (queryrangebase.Response, error) { args := m.Called(ctx, req) - return args.Get(0).(*httpgrpc.HTTPResponse), args.Error(1) + return args.Get(0).(queryrangebase.Response), args.Error(1) } diff --git a/pkg/querier/worker/util.go b/pkg/querier/worker/util.go index e97a000eef..c0473319cf 100644 --- a/pkg/querier/worker/util.go +++ b/pkg/querier/worker/util.go @@ -4,11 +4,15 @@ package worker import ( "context" + "net/http" "time" "github.com/go-kit/log" "github.com/go-kit/log/level" + "github.com/grafana/dskit/httpgrpc" "go.uber.org/atomic" + + "github.com/grafana/loki/pkg/querier/queryrange" ) // newExecutionContext returns a new execution context (execCtx) that wraps the input workerCtx and @@ -77,3 +81,44 @@ func newExecutionContext(workerCtx context.Context, logger log.Logger) (execCtx return } + +// handle converts the request and applies it to the handler. +func handle(ctx context.Context, request *httpgrpc.HTTPRequest, handler RequestHandler, codec GRPCCodec) *httpgrpc.HTTPResponse { + req, ctx, err := codec.DecodeHTTPGrpcRequest(ctx, request) + if err != nil { + response, ok := httpgrpc.HTTPResponseFromError(err) + if !ok { + return &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + } + return response + } + + resp, err := handler.Do(ctx, req) + if err != nil { + response, ok := httpgrpc.HTTPResponseFromError(err) + if !ok { + return &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + } + return response + } + + response, err := queryrange.DefaultCodec.EncodeHTTPGrpcResponse(ctx, request, resp) + if err != nil { + response, ok := httpgrpc.HTTPResponseFromError(err) + if !ok { + return &httpgrpc.HTTPResponse{ + Code: http.StatusInternalServerError, + Body: []byte(err.Error()), + } + } + return response + } + + return response +} diff --git a/pkg/querier/worker/worker.go b/pkg/querier/worker/worker.go index 7c88ac92b8..1e9974784e 100644 --- a/pkg/querier/worker/worker.go +++ b/pkg/querier/worker/worker.go @@ -17,6 +17,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" "github.com/grafana/loki/pkg/util" lokiutil "github.com/grafana/loki/pkg/util" ) @@ -50,7 +51,13 @@ func (cfg *Config) Validate() error { // Handler for HTTP requests wrapped in protobuf messages. type RequestHandler interface { - Handle(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) + Do(context.Context, queryrangebase.Request) (queryrangebase.Response, error) +} + +// Decodes httpgrpc.HTTPRequests to queryrangebase.Requests. This is used by the +// frontend and scheduler processor for backwards compatibility. +type GRPCCodec interface { + DecodeHTTPGrpcRequest(context.Context, *httpgrpc.HTTPRequest) (queryrangebase.Request, context.Context, error) } // Single processor handles all streaming operations to query-frontend or query-scheduler to fetch queries @@ -86,7 +93,7 @@ type querierWorker struct { metrics *Metrics } -func NewQuerierWorker(cfg Config, rng ring.ReadRing, handler RequestHandler, logger log.Logger, reg prometheus.Registerer) (services.Service, error) { +func NewQuerierWorker(cfg Config, rng ring.ReadRing, handler RequestHandler, logger log.Logger, reg prometheus.Registerer, codec GRPCCodec) (services.Service, error) { if cfg.QuerierID == "" { hostname, err := os.Hostname() if err != nil { @@ -103,18 +110,18 @@ func NewQuerierWorker(cfg Config, rng ring.ReadRing, handler RequestHandler, log switch { case rng != nil: level.Info(logger).Log("msg", "Starting querier worker using query-scheduler and scheduler ring for addresses") - processor, servs = newSchedulerProcessor(cfg, handler, logger, metrics) + processor, servs = newSchedulerProcessor(cfg, handler, logger, metrics, codec) case cfg.SchedulerAddress != "": level.Info(logger).Log("msg", "Starting querier worker connected to query-scheduler", "scheduler", cfg.SchedulerAddress) address = cfg.SchedulerAddress - processor, servs = newSchedulerProcessor(cfg, handler, logger, metrics) + processor, servs = newSchedulerProcessor(cfg, handler, logger, metrics, codec) case cfg.FrontendAddress != "": level.Info(logger).Log("msg", "Starting querier worker connected to query-frontend", "frontend", cfg.FrontendAddress) address = cfg.FrontendAddress - processor = newFrontendProcessor(cfg, handler, logger) + processor = newFrontendProcessor(cfg, handler, logger, codec) default: return nil, errors.New("unable to start the querier worker, need to configure one of frontend_address, scheduler_address, or a ring config in the query_scheduler config block") } diff --git a/pkg/querier/worker_service.go b/pkg/querier/worker_service.go index 7565c8f3cb..1601620a48 100644 --- a/pkg/querier/worker_service.go +++ b/pkg/querier/worker_service.go @@ -2,22 +2,15 @@ package querier import ( "fmt" - "net/http" "github.com/go-kit/log/level" - "github.com/gorilla/mux" - httpgrpc_server "github.com/grafana/dskit/httpgrpc/server" - "github.com/grafana/dskit/middleware" "github.com/grafana/dskit/ring" "github.com/grafana/dskit/services" - "github.com/opentracing-contrib/go-stdlib/nethttp" - "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" querier_worker "github.com/grafana/loki/pkg/querier/worker" - "github.com/grafana/loki/pkg/util/httpreq" util_log "github.com/grafana/loki/pkg/util/log" - serverutil "github.com/grafana/loki/pkg/util/server" ) type WorkerServiceConfig struct { @@ -25,12 +18,27 @@ type WorkerServiceConfig struct { ReadEnabled bool GrpcListenAddress string GrpcListenPort int + QuerierMaxConcurrent int QuerierWorkerConfig *querier_worker.Config QueryFrontendEnabled bool QuerySchedulerEnabled bool SchedulerRing ring.ReadRing } +func (cfg WorkerServiceConfig) QuerierRunningStandalone() bool { + runningStandalone := !cfg.QueryFrontendEnabled && !cfg.QuerySchedulerEnabled && !cfg.ReadEnabled && !cfg.AllEnabled + level.Debug(util_log.Logger).Log( + "msg", "determining if querier is running as standalone target", + "runningStandalone", runningStandalone, + "queryFrontendEnabled", cfg.QueryFrontendEnabled, + "queryScheduleEnabled", cfg.QuerySchedulerEnabled, + "readEnabled", cfg.ReadEnabled, + "allEnabled", cfg.AllEnabled, + ) + + return runningStandalone +} + // InitWorkerService takes a config object, a map of routes to handlers, an external http router and external // http handler, and an auth middleware wrapper. This function creates an internal HTTP router that responds to all // the provided query routes/handlers. This router can either be registered with the external Loki HTTP server, or @@ -46,55 +54,15 @@ type WorkerServiceConfig struct { func InitWorkerService( cfg WorkerServiceConfig, reg prometheus.Registerer, - queryRouterPathPrefix string, - queryRoutesToHandlers map[string]http.Handler, - alwaysExternalRoutesToHandlers map[string]http.Handler, - externalRouter *mux.Router, - externalHandler http.Handler, - authMiddleware middleware.Interface, + handler queryrangebase.Handler, + codec querier_worker.GRPCCodec, ) (serve services.Service, err error) { - // Create a couple Middlewares used to handle panics, perform auth, parse forms in http request, and set content type in response - handlerMiddleware := middleware.Merge( - httpreq.ExtractQueryTagsMiddleware(), - serverutil.RecoveryHTTPMiddleware, - authMiddleware, - serverutil.NewPrepopulateMiddleware(), - serverutil.ResponseJSONMiddleware(), - ) - - internalRouter := mux.NewRouter() - if queryRouterPathPrefix != "" { - internalRouter = internalRouter.PathPrefix(queryRouterPathPrefix).Subrouter() - } - for route, handler := range queryRoutesToHandlers { - internalRouter.Path(route).Methods("GET", "POST").Handler(handler) - } - - // There are some routes which are always registered on the external router, add them now and - // wrap them with the externalMiddleware - for route, handler := range alwaysExternalRoutesToHandlers { - externalRouter.Path(route).Methods("GET", "POST").Handler(handlerMiddleware.Wrap(handler)) - } - // If the querier is running standalone without the query-frontend or query-scheduler, we must register the internal // HTTP handler externally (as it's the only handler that needs to register on querier routes) and provide the // external Loki Server HTTP handler to the frontend worker to ensure requests it processes use the default // middleware instrumentation. - if querierRunningStandalone(cfg) { - - // First, register the internal querier handler with the external HTTP server - routes := make([]string, len(queryRoutesToHandlers)) - var idx = 0 - for route := range queryRoutesToHandlers { - routes[idx] = route - idx++ - } - - // Register routes externally - for _, route := range routes { - externalRouter.Path(route).Methods("GET", "POST").Handler(handlerMiddleware.Wrap(internalRouter)) - } + if cfg.QuerierRunningStandalone() { //If no scheduler ring or frontend or scheduler address has been configured, then there is no place for the //querier worker to request work from, so no need to start a worker service @@ -107,9 +75,10 @@ func InitWorkerService( return querier_worker.NewQuerierWorker( *(cfg.QuerierWorkerConfig), cfg.SchedulerRing, - httpgrpc_server.NewServer(externalHandler), + handler, util_log.Logger, reg, + codec, ) } @@ -127,38 +96,14 @@ func InitWorkerService( cfg.QuerierWorkerConfig.FrontendAddress = address } - // Add a middleware to extract the trace context and add a header. - var internalHandler http.Handler - internalHandler = nethttp.MiddlewareFunc( - opentracing.GlobalTracer(), - internalRouter.ServeHTTP, - nethttp.OperationNameFunc(func(r *http.Request) string { - return "internalQuerier" - })) - - internalHandler = handlerMiddleware.Wrap(internalHandler) - //Return a querier worker pointed to the internal querier HTTP handler so there is not a conflict in routes between the querier //and the query frontend return querier_worker.NewQuerierWorker( *(cfg.QuerierWorkerConfig), cfg.SchedulerRing, - httpgrpc_server.NewServer(internalHandler), + handler, util_log.Logger, reg, + codec, ) } - -func querierRunningStandalone(cfg WorkerServiceConfig) bool { - runningStandalone := !cfg.QueryFrontendEnabled && !cfg.QuerySchedulerEnabled && !cfg.ReadEnabled && !cfg.AllEnabled - level.Debug(util_log.Logger).Log( - "msg", "determining if querier is running as standalone target", - "runningStandalone", runningStandalone, - "queryFrontendEnabled", cfg.QueryFrontendEnabled, - "queryScheduleEnabled", cfg.QuerySchedulerEnabled, - "readEnabled", cfg.ReadEnabled, - "allEnabled", cfg.AllEnabled, - ) - - return runningStandalone -} diff --git a/pkg/querier/worker_service_test.go b/pkg/querier/worker_service_test.go index e4f025e657..a1fd9c0db3 100644 --- a/pkg/querier/worker_service_test.go +++ b/pkg/querier/worker_service_test.go @@ -1,247 +1,239 @@ package querier import ( - "net/http" - "net/http/httptest" + //"context" + //"net/http" + //"net/http/httptest" "testing" + /* + "github.com/gorilla/mux" + "github.com/grafana/dskit/middleware" + "github.com/grafana/dskit/services" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - "github.com/gorilla/mux" - "github.com/grafana/dskit/middleware" - "github.com/grafana/dskit/services" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - querier_worker "github.com/grafana/loki/pkg/querier/worker" -) + "github.com/grafana/loki/pkg/querier/queryrange/queryrangebase" + querier_worker "github.com/grafana/loki/pkg/querier/worker" + */) func Test_InitQuerierService(t *testing.T) { - var mockQueryHandlers = map[string]http.Handler{ - "/loki/api/v1/query": http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - _, err := res.Write([]byte(`{"handler": "test"}`)) - res.Header().Del("Content-Type") - require.NoError(t, err) - }), - } - - var alwaysExternalHandlers = map[string]http.Handler{ - "/loki/api/v1/tail": http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - _, err := res.Write([]byte("test tail handler")) - require.NoError(t, err) - }), - } - - testContext := func(config WorkerServiceConfig, authMiddleware middleware.Interface) (*mux.Router, services.Service) { - externalRouter := mux.NewRouter() - - if authMiddleware == nil { - authMiddleware = middleware.Identity - } - - pathPrefix := "" - querierWorkerService, err := InitWorkerService( - config, - nil, - pathPrefix, - mockQueryHandlers, - alwaysExternalHandlers, - externalRouter, - http.HandlerFunc(externalRouter.ServeHTTP), - authMiddleware, - ) - require.NoError(t, err) - - return externalRouter, querierWorkerService - } - - t.Run("when querier is configured to run standalone, without a query frontend", func(t *testing.T) { - t.Run("register the internal query handlers externally", func(t *testing.T) { - config := WorkerServiceConfig{ - QueryFrontendEnabled: false, - QuerySchedulerEnabled: false, - AllEnabled: false, - QuerierWorkerConfig: &querier_worker.Config{}, - } - - externalRouter, _ := testContext(config, nil) - - recorder := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) - externalRouter.ServeHTTP(recorder, request) - assert.Equal(t, 200, recorder.Code) - assert.Equal(t, `{"handler": "test"}`, recorder.Body.String()) - // Tail endpoints always external - recorder = httptest.NewRecorder() - request = httptest.NewRequest("GET", "/loki/api/v1/tail", nil) - externalRouter.ServeHTTP(recorder, request) - assert.Equal(t, 200, recorder.Code) - assert.Equal(t, "test tail handler", recorder.Body.String()) - }) - - t.Run("wrap external handler with auth middleware", func(t *testing.T) { - config := WorkerServiceConfig{ - QueryFrontendEnabled: false, - QuerySchedulerEnabled: false, - AllEnabled: false, - QuerierWorkerConfig: &querier_worker.Config{}, + // TODO: use in modules test + /* + var alwaysExternalHandlers = map[string]http.Handler{ + "/loki/api/v1/tail": http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + _, err := res.Write([]byte("test tail handler")) + require.NoError(t, err) + }), } - requestedAuthenticated := false - mockAuthMiddleware := middleware.Func(func(next http.Handler) http.Handler { - requestedAuthenticated = true - return next - }) - - externalRouter, _ := testContext(config, mockAuthMiddleware) - - recorder := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) - externalRouter.ServeHTTP(recorder, request) - assert.True(t, requestedAuthenticated) + handler := queryrangebase.HandlerFunc(func(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) { + return nil, nil }) - t.Run("wrap external handler with response json middleware", func(t *testing.T) { - // note: this test only assures that the content type of the response is - // set if the handler function does not override it, which happens in the - // actual implementation, see - // https://github.com/grafana/loki/blob/34a012adcfade43291de3a7670f53679ea06aefe/pkg/lokifrontend/frontend/transport/handler.go#L136-L139 - config := WorkerServiceConfig{ - QueryFrontendEnabled: false, - QuerySchedulerEnabled: false, - AllEnabled: false, - QuerierWorkerConfig: &querier_worker.Config{}, - } + testContext := func(config WorkerServiceConfig) (*mux.Router, services.Service) { + externalRouter := mux.NewRouter() - externalRouter, _ := testContext(config, nil) - - recorder := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) - externalRouter.ServeHTTP(recorder, request) - - contentTypeHeader := recorder.Header().Get("Content-Type") - assert.Equal(t, "application/json; charset=UTF-8", contentTypeHeader) - }) + querierWorkerService, err := InitWorkerService( + config, + nil, + handler, + ) + require.NoError(t, err) - t.Run("do not create a querier worker service if neither frontend address nor scheduler address has been configured", func(t *testing.T) { - config := WorkerServiceConfig{ - QueryFrontendEnabled: false, - QuerySchedulerEnabled: false, - AllEnabled: false, - QuerierWorkerConfig: &querier_worker.Config{}, - } + return externalRouter, querierWorkerService + } - _, workerService := testContext(config, nil) - assert.Nil(t, workerService) - }) + t.Run("when querier is configured to run standalone, without a query frontend", func(t *testing.T) { + t.Run("register the internal query handlers externally", func(t *testing.T) { + config := WorkerServiceConfig{ + QueryFrontendEnabled: false, + QuerySchedulerEnabled: false, + AllEnabled: false, + QuerierWorkerConfig: &querier_worker.Config{}, + } + + externalRouter, _ := testContext(config) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) + externalRouter.ServeHTTP(recorder, request) + assert.Equal(t, 200, recorder.Code) + assert.Equal(t, `{"handler": "test"}`, recorder.Body.String()) + + // Tail endpoints always external + recorder = httptest.NewRecorder() + request = httptest.NewRequest("GET", "/loki/api/v1/tail", nil) + externalRouter.ServeHTTP(recorder, request) + assert.Equal(t, 200, recorder.Code) + assert.Equal(t, "test tail handler", recorder.Body.String()) + }) + + t.Run("wrap external handler with auth middleware", func(t *testing.T) { + config := WorkerServiceConfig{ + QueryFrontendEnabled: false, + QuerySchedulerEnabled: false, + AllEnabled: false, + QuerierWorkerConfig: &querier_worker.Config{}, + } + + requestedAuthenticated := false + mockAuthMiddleware := middleware.Func(func(next http.Handler) http.Handler { + requestedAuthenticated = true + return next + }) + + externalRouter, _ := testContext(config, mockAuthMiddleware) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) + externalRouter.ServeHTTP(recorder, request) + assert.True(t, requestedAuthenticated) + }) + + t.Run("wrap external handler with response json middleware", func(t *testing.T) { + // note: this test only assures that the content type of the response is + // set if the handler function does not override it, which happens in the + // actual implementation, see + // https://github.com/grafana/loki/blob/34a012adcfade43291de3a7670f53679ea06aefe/pkg/lokifrontend/frontend/transport/handler.go#L136-L139 + config := WorkerServiceConfig{ + QueryFrontendEnabled: false, + QuerySchedulerEnabled: false, + AllEnabled: false, + QuerierWorkerConfig: &querier_worker.Config{}, + } + + externalRouter, _ := testContext(config, nil) + + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) + externalRouter.ServeHTTP(recorder, request) + + contentTypeHeader := recorder.Header().Get("Content-Type") + assert.Equal(t, "application/json; charset=UTF-8", contentTypeHeader) + }) + + t.Run("do not create a querier worker service if neither frontend address nor scheduler address has been configured", func(t *testing.T) { + config := WorkerServiceConfig{ + QueryFrontendEnabled: false, + QuerySchedulerEnabled: false, + AllEnabled: false, + QuerierWorkerConfig: &querier_worker.Config{}, + } + + _, workerService := testContext(config, nil) + assert.Nil(t, workerService) + }) + + t.Run("return a querier worker service if frontend or scheduler address has been configured", func(t *testing.T) { + withFrontendConfig := WorkerServiceConfig{ + QuerierWorkerConfig: &querier_worker.Config{ + FrontendAddress: "http://example.com", + }, + } + withSchedulerConfig := WorkerServiceConfig{ + QuerierWorkerConfig: &querier_worker.Config{ + SchedulerAddress: "http://example.com", + }, + } + + for _, config := range []WorkerServiceConfig{ + withFrontendConfig, + withSchedulerConfig, + } { + _, workerService := testContext(config, nil) + assert.NotNil(t, workerService) + } + }) + }) - t.Run("return a querier worker service if frontend or scheduler address has been configured", func(t *testing.T) { - withFrontendConfig := WorkerServiceConfig{ - QuerierWorkerConfig: &querier_worker.Config{ - FrontendAddress: "http://example.com", + t.Run("when query frontend, scheduler, or all target is enabled", func(t *testing.T) { + defaultWorkerConfig := querier_worker.Config{} + nonStandaloneTargetPermutations := []WorkerServiceConfig{ + { + QueryFrontendEnabled: true, + QuerySchedulerEnabled: false, + AllEnabled: false, + QuerierWorkerConfig: &defaultWorkerConfig, }, - } - withSchedulerConfig := WorkerServiceConfig{ - QuerierWorkerConfig: &querier_worker.Config{ - SchedulerAddress: "http://example.com", + { + QueryFrontendEnabled: false, + QuerySchedulerEnabled: true, + AllEnabled: false, + QuerierWorkerConfig: &defaultWorkerConfig, + }, + { + QueryFrontendEnabled: false, + QuerySchedulerEnabled: false, + AllEnabled: true, + QuerierWorkerConfig: &defaultWorkerConfig, + }, + { + QueryFrontendEnabled: true, + QuerySchedulerEnabled: true, + AllEnabled: false, + QuerierWorkerConfig: &defaultWorkerConfig, + }, + { + QueryFrontendEnabled: true, + QuerySchedulerEnabled: false, + AllEnabled: true, + QuerierWorkerConfig: &defaultWorkerConfig, + }, + { + QueryFrontendEnabled: false, + QuerySchedulerEnabled: true, + AllEnabled: true, + QuerierWorkerConfig: &defaultWorkerConfig, + }, + { + QueryFrontendEnabled: true, + QuerySchedulerEnabled: true, + AllEnabled: true, + QuerierWorkerConfig: &defaultWorkerConfig, }, } - for _, config := range []WorkerServiceConfig{ - withFrontendConfig, - withSchedulerConfig, - } { - _, workerService := testContext(config, nil) - assert.NotNil(t, workerService) - } - }) - }) - - t.Run("when query frontend, scheduler, or all target is enabled", func(t *testing.T) { - defaultWorkerConfig := querier_worker.Config{} - nonStandaloneTargetPermutations := []WorkerServiceConfig{ - { - QueryFrontendEnabled: true, - QuerySchedulerEnabled: false, - AllEnabled: false, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: false, - QuerySchedulerEnabled: true, - AllEnabled: false, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: false, - QuerySchedulerEnabled: false, - AllEnabled: true, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: true, - QuerySchedulerEnabled: true, - AllEnabled: false, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: true, - QuerySchedulerEnabled: false, - AllEnabled: true, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: false, - QuerySchedulerEnabled: true, - AllEnabled: true, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - { - QueryFrontendEnabled: true, - QuerySchedulerEnabled: true, - AllEnabled: true, - QuerierWorkerConfig: &defaultWorkerConfig, - }, - } - - t.Run("do not register the internal query handler externally", func(t *testing.T) { - for _, config := range nonStandaloneTargetPermutations { - externalRouter, _ := testContext(config, nil) - recorder := httptest.NewRecorder() - request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) - externalRouter.ServeHTTP(recorder, request) - assert.Equal(t, 404, recorder.Code) - - // Tail endpoints always external - recorder = httptest.NewRecorder() - request = httptest.NewRequest("GET", "/loki/api/v1/tail", nil) - externalRouter.ServeHTTP(recorder, request) - assert.Equal(t, 200, recorder.Code) - assert.Equal(t, "test tail handler", recorder.Body.String()) - } - }) + t.Run("do not register the internal query handler externally", func(t *testing.T) { + for _, config := range nonStandaloneTargetPermutations { + externalRouter, _ := testContext(config) + recorder := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/loki/api/v1/query", nil) + externalRouter.ServeHTTP(recorder, request) + assert.Equal(t, 404, recorder.Code) + + // Tail endpoints always external + recorder = httptest.NewRecorder() + request = httptest.NewRequest("GET", "/loki/api/v1/tail", nil) + externalRouter.ServeHTTP(recorder, request) + assert.Equal(t, 200, recorder.Code) + assert.Equal(t, "test tail handler", recorder.Body.String()) + } + }) - t.Run("use localhost as the worker address if none is set", func(t *testing.T) { - for _, config := range nonStandaloneTargetPermutations { - workerConfig := querier_worker.Config{} - config.QuerierWorkerConfig = &workerConfig - config.GrpcListenPort = 1234 + t.Run("use localhost as the worker address if none is set", func(t *testing.T) { + for _, config := range nonStandaloneTargetPermutations { + workerConfig := querier_worker.Config{} + config.QuerierWorkerConfig = &workerConfig + config.GrpcListenPort = 1234 - testContext(config, nil) + testContext(config) - assert.Equal(t, "127.0.0.1:1234", workerConfig.FrontendAddress) - } - }) + assert.Equal(t, "127.0.0.1:1234", workerConfig.FrontendAddress) + } + }) - t.Run("always return a query worker service", func(t *testing.T) { - for _, config := range nonStandaloneTargetPermutations { - workerConfig := querier_worker.Config{} - config.QuerierWorkerConfig = &workerConfig - config.GrpcListenPort = 1234 + t.Run("always return a query worker service", func(t *testing.T) { + for _, config := range nonStandaloneTargetPermutations { + workerConfig := querier_worker.Config{} + config.QuerierWorkerConfig = &workerConfig + config.GrpcListenPort = 1234 - _, querierWorkerService := testContext(config, nil) + _, querierWorkerService := testContext(config) - assert.NotNil(t, querierWorkerService) - } + assert.NotNil(t, querierWorkerService) + } + }) }) - }) + */ } diff --git a/pkg/storage/chunk/client/congestion/congestion.go b/pkg/storage/chunk/client/congestion/congestion.go index cc0ed7b739..3e591dd121 100644 --- a/pkg/storage/chunk/client/congestion/congestion.go +++ b/pkg/storage/chunk/client/congestion/congestion.go @@ -17,32 +17,32 @@ func NewController(cfg Config, logger log.Logger, metrics *Metrics) Controller { } func newController(cfg Config, logger log.Logger) Controller { - strat := strings.ToLower(cfg.Controller.Strategy) - switch strat { + start := strings.ToLower(cfg.Controller.Strategy) + switch start { case "aimd": return NewAIMDController(cfg).withLogger(logger) default: - level.Warn(logger).Log("msg", "unrecognized congestion control strategy in config, using noop", "strategy", strat) + level.Warn(logger).Log("msg", "unrecognized congestion control strategy in config, using noop", "strategy", start) return NewNoopController(cfg).withLogger(logger) } } func newRetrier(cfg Config, logger log.Logger) Retrier { - strat := strings.ToLower(cfg.Retry.Strategy) - switch strat { + start := strings.ToLower(cfg.Retry.Strategy) + switch start { case "limited": return NewLimitedRetrier(cfg).withLogger(logger) default: - level.Warn(logger).Log("msg", "unrecognized retried strategy in config, using noop", "strategy", strat) + level.Warn(logger).Log("msg", "unrecognized retried strategy in config, using noop", "strategy", start) return NewNoopRetrier(cfg).withLogger(logger) } } func newHedger(cfg Config, logger log.Logger) Hedger { - strat := strings.ToLower(cfg.Hedge.Strategy) - switch strat { + start := strings.ToLower(cfg.Hedge.Strategy) + switch start { default: - level.Warn(logger).Log("msg", "unrecognized hedging strategy in config, using noop", "strategy", strat) + level.Warn(logger).Log("msg", "unrecognized hedging strategy in config, using noop", "strategy", start) return NewNoopHedger(cfg).withLogger(logger) } } diff --git a/pkg/storage/chunk/client/congestion/controller_test.go b/pkg/storage/chunk/client/congestion/controller_test.go index dcd311bc41..c65f1333e9 100644 --- a/pkg/storage/chunk/client/congestion/controller_test.go +++ b/pkg/storage/chunk/client/congestion/controller_test.go @@ -266,8 +266,8 @@ func (m *mockObjectClient) IsObjectNotFoundErr(error) bool { return false } func (m *mockObjectClient) IsRetryableErr(error) bool { return !m.nonRetryableErrs } func (m *mockObjectClient) Stop() {} -func newMockObjectClient(strat requestFailer) *mockObjectClient { - return &mockObjectClient{strategy: strat} +func newMockObjectClient(start requestFailer) *mockObjectClient { + return &mockObjectClient{strategy: start} } type requestFailer interface { diff --git a/pkg/util/marshal/marshal.go b/pkg/util/marshal/marshal.go index 6298cf32d1..be03bed347 100644 --- a/pkg/util/marshal/marshal.go +++ b/pkg/util/marshal/marshal.go @@ -9,12 +9,14 @@ import ( "github.com/gorilla/websocket" jsoniter "github.com/json-iterator/go" + "github.com/prometheus/prometheus/promql/parser" "github.com/grafana/loki/pkg/loghttp" legacy "github.com/grafana/loki/pkg/loghttp/legacy" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logqlmodel" - "github.com/grafana/loki/pkg/storage/stores/index/stats" + "github.com/grafana/loki/pkg/logqlmodel/stats" + indexStats "github.com/grafana/loki/pkg/storage/stores/index/stats" marshal_legacy "github.com/grafana/loki/pkg/util/marshal/legacy" ) @@ -23,20 +25,20 @@ func WriteResponseJSON(r *http.Request, v any, w http.ResponseWriter) error { case logqlmodel.Result: version := loghttp.GetVersion(r.RequestURI) if version == loghttp.VersionV1 { - return WriteQueryResponseJSON(result, w) + return WriteQueryResponseJSON(result.Data, result.Statistics, w) } return marshal_legacy.WriteQueryResponseJSON(result, w) case *logproto.LabelResponse: version := loghttp.GetVersion(r.RequestURI) if version == loghttp.VersionV1 { - return WriteLabelResponseJSON(*result, w) + return WriteLabelResponseJSON(result.GetValues(), w) } return marshal_legacy.WriteLabelResponseJSON(*result, w) case *logproto.SeriesResponse: - return WriteSeriesResponseJSON(*result, w) - case *stats.Stats: + return WriteSeriesResponseJSON(result.GetSeries(), w) + case *indexStats.Stats: return WriteIndexStatsResponseJSON(result, w) case *logproto.VolumeResponse: return WriteVolumeResponseJSON(result, w) @@ -46,10 +48,10 @@ func WriteResponseJSON(r *http.Request, v any, w http.ResponseWriter) error { // WriteQueryResponseJSON marshals the promql.Value to v1 loghttp JSON and then // writes it to the provided io.Writer. -func WriteQueryResponseJSON(v logqlmodel.Result, w io.Writer) error { +func WriteQueryResponseJSON(data parser.Value, statistics stats.Result, w io.Writer) error { s := jsoniter.ConfigFastest.BorrowStream(w) defer jsoniter.ConfigFastest.ReturnStream(s) - err := EncodeResult(v, s) + err := EncodeResult(data, statistics, s) if err != nil { return fmt.Errorf("could not write JSON response: %w", err) } @@ -59,10 +61,10 @@ func WriteQueryResponseJSON(v logqlmodel.Result, w io.Writer) error { // WriteLabelResponseJSON marshals a logproto.LabelResponse to v1 loghttp JSON // and then writes it to the provided io.Writer. -func WriteLabelResponseJSON(l logproto.LabelResponse, w io.Writer) error { +func WriteLabelResponseJSON(data []string, w io.Writer) error { v1Response := loghttp.LabelResponse{ Status: "success", - Data: l.GetValues(), + Data: data, } s := jsoniter.ConfigFastest.BorrowStream(w) @@ -93,13 +95,13 @@ func WriteTailResponseJSON(r legacy.TailResponse, c WebsocketWriter) error { // WriteSeriesResponseJSON marshals a logproto.SeriesResponse to v1 loghttp JSON and then // writes it to the provided io.Writer. -func WriteSeriesResponseJSON(r logproto.SeriesResponse, w io.Writer) error { +func WriteSeriesResponseJSON(series []logproto.SeriesIdentifier, w io.Writer) error { adapter := &seriesResponseAdapter{ Status: "success", - Data: make([]map[string]string, 0, len(r.GetSeries())), + Data: make([]map[string]string, 0, len(series)), } - for _, series := range r.GetSeries() { + for _, series := range series { adapter.Data = append(adapter.Data, series.GetLabels()) } @@ -119,7 +121,7 @@ type seriesResponseAdapter struct { // WriteIndexStatsResponseJSON marshals a gatewaypb.Stats to JSON and then // writes it to the provided io.Writer. -func WriteIndexStatsResponseJSON(r *stats.Stats, w io.Writer) error { +func WriteIndexStatsResponseJSON(r *indexStats.Stats, w io.Writer) error { s := jsoniter.ConfigFastest.BorrowStream(w) defer jsoniter.ConfigFastest.ReturnStream(s) s.WriteVal(r) diff --git a/pkg/util/marshal/marshal_test.go b/pkg/util/marshal/marshal_test.go index ec25104cc2..dc80438d1b 100644 --- a/pkg/util/marshal/marshal_test.go +++ b/pkg/util/marshal/marshal_test.go @@ -19,6 +19,7 @@ import ( legacy "github.com/grafana/loki/pkg/loghttp/legacy" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logqlmodel/stats" ) // covers responses from /loki/api/v1/query_range and /loki/api/v1/query @@ -600,7 +601,7 @@ var tailTests = []struct { func Test_WriteQueryResponseJSON(t *testing.T) { for i, queryTest := range queryTests { var b bytes.Buffer - err := WriteQueryResponseJSON(logqlmodel.Result{Data: queryTest.actual}, &b) + err := WriteQueryResponseJSON(queryTest.actual, stats.Result{}, &b) require.NoError(t, err) require.JSONEqf(t, queryTest.expected, b.String(), "Query Test %d failed", i) @@ -610,7 +611,7 @@ func Test_WriteQueryResponseJSON(t *testing.T) { func Test_WriteLabelResponseJSON(t *testing.T) { for i, labelTest := range labelTests { var b bytes.Buffer - err := WriteLabelResponseJSON(labelTest.actual, &b) + err := WriteLabelResponseJSON(labelTest.actual.GetValues(), &b) require.NoError(t, err) require.JSONEqf(t, labelTest.expected, b.String(), "Label Test %d failed", i) @@ -632,7 +633,7 @@ func Test_WriteQueryResponseJSONWithError(t *testing.T) { }, } var b bytes.Buffer - err := WriteQueryResponseJSON(broken, &b) + err := WriteQueryResponseJSON(broken.Data, stats.Result{}, &b) require.Error(t, err) } @@ -747,7 +748,7 @@ func Test_WriteSeriesResponseJSON(t *testing.T) { } { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { var b bytes.Buffer - err := WriteSeriesResponseJSON(tc.input, &b) + err := WriteSeriesResponseJSON(tc.input.GetSeries(), &b) require.NoError(t, err) require.JSONEqf(t, tc.expected, b.String(), "Series Test %d failed", i) @@ -882,7 +883,7 @@ func Benchmark_Encode(b *testing.B) { for n := 0; n < b.N; n++ { for _, queryTest := range queryTests { - require.NoError(b, WriteQueryResponseJSON(logqlmodel.Result{Data: queryTest.actual}, buf)) + require.NoError(b, WriteQueryResponseJSON(queryTest.actual, stats.Result{}, buf)) buf.Reset() } } diff --git a/pkg/util/marshal/query.go b/pkg/util/marshal/query.go index f580aa70b9..56cc567246 100644 --- a/pkg/util/marshal/query.go +++ b/pkg/util/marshal/query.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/loki/pkg/loghttp" "github.com/grafana/loki/pkg/logproto" "github.com/grafana/loki/pkg/logqlmodel" + "github.com/grafana/loki/pkg/logqlmodel/stats" ) // NewResultValue constructs a ResultValue from a promql.Value @@ -173,14 +174,14 @@ func NewMetric(l labels.Labels) model.Metric { return ret } -func EncodeResult(v logqlmodel.Result, s *jsoniter.Stream) error { +func EncodeResult(data parser.Value, statistics stats.Result, s *jsoniter.Stream) error { s.WriteObjectStart() s.WriteObjectField("status") s.WriteString("success") s.WriteMore() s.WriteObjectField("data") - err := encodeData(v, s) + err := encodeData(data, statistics, s) if err != nil { return err } @@ -189,22 +190,22 @@ func EncodeResult(v logqlmodel.Result, s *jsoniter.Stream) error { return nil } -func encodeData(v logqlmodel.Result, s *jsoniter.Stream) error { +func encodeData(data parser.Value, statistics stats.Result, s *jsoniter.Stream) error { s.WriteObjectStart() s.WriteObjectField("resultType") - s.WriteString(string(v.Data.Type())) + s.WriteString(string(data.Type())) s.WriteMore() s.WriteObjectField("result") - err := encodeResult(v.Data, s) + err := encodeResult(data, s) if err != nil { return err } s.WriteMore() s.WriteObjectField("stats") - s.WriteVal(v.Statistics) + s.WriteVal(statistics) s.WriteObjectEnd() s.Flush()