From 2b66c6b12601ea58cef02e24f3aef3568b87d814 Mon Sep 17 00:00:00 2001 From: Ed Welch Date: Thu, 5 Jan 2023 12:32:45 -0500 Subject: [PATCH] Loki: add a 'since' query parameter for easier querying using relative time (#7964) Signed-off-by: Edward Welch **What this PR does / why we need it**: logcli has a command line parameter called `since` which makes it easy to query logs relative to the current time, however this is some syntactic sugar used in logcli to calculate `start` and `end` query parameters for the query sent to Loki. This PR adds native support for a `since` query parameter to make it a little easier to consume the Loki query API without having to calculate precise start/end times with every request. This is useful if you are polling the Loki API from a third party app which doesn't let you easily build dynamic URL's and all you want to do is poll for say the last 3 hours of logs. **Which issue(s) this PR fixes**: Fixes # **Special notes for your reviewer**: **Checklist** - [x] Reviewed the `CONTRIBUTING.md` guide - [x] Documentation added - [x] Tests updated - [x] `CHANGELOG.md` updated - [ ] Changes that require user attention or interaction to upgrade are documented in `docs/sources/upgrading/_index.md` Signed-off-by: Edward Welch Co-authored-by: Owen Diehl --- CHANGELOG.md | 1 + docs/sources/api/_index.md | 7 ++ pkg/loghttp/params.go | 34 +++++++-- pkg/loghttp/params_test.go | 142 +++++++++++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad38bfc79..12ac0e3eb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ * [7804](https://github.com/grafana/loki/pull/7804) **sandeepsukhani**: Use grpc for communicating with compactor for query time filtering of data requested for deletion. * [7817](https://github.com/grafana/loki/pull/7817) **kavirajk**: fix(memcached): panic on send on closed channel. * [7916](https://github.com/grafana/loki/pull/7916) **ssncferreira**: Add `doc-generator` tool to generate configuration flags documentation. +* [7964](https://github.com/grafana/loki/pull/7964) **slim-bean**: Add a `since` query parameter to allow querying based on relative time. * [7989](https://github.com/grafana/loki/pull/7989) **liguozhong**: logql support `sort` and `sort_desc`. * [7997](https://github.com/grafana/loki/pull/7997) **kavirajk**: fix(promtail): Fix cri tags extra new lines when joining partial lines * [8027](https://github.com/grafana/loki/pull/8027) **kavirajk**: chore(promtail): Make `batchwait` and `batchsize` config explicit with yaml tags diff --git a/docs/sources/api/_index.md b/docs/sources/api/_index.md index 4f00864a6c..413de4a334 100644 --- a/docs/sources/api/_index.md +++ b/docs/sources/api/_index.md @@ -269,6 +269,7 @@ accepts the following query parameters in the URL: - `limit`: The max number of entries to return. It defaults to `100`. Only applies to query types which produce a stream(log lines) response. - `start`: The start time for the query as a nanosecond Unix epoch or another [supported format](#timestamp-formats). Defaults to one hour ago. - `end`: The end time for the query as a nanosecond Unix epoch or another [supported format](#timestamp-formats). Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. - `step`: Query resolution step width in `duration` format or float number of seconds. `duration` refers to Prometheus duration strings of the form `[0-9]+[smhdwy]`. For example, 5m refers to a duration of 5 minutes. Defaults to a dynamic value based on `start` and `end`. Only applies to query types which produce a matrix response. - `interval`: This parameter is experimental; see the explanation under Step versus interval. Only return entries at (or greater than) the specified interval, can be a `duration` format or float number of seconds. Only applies to queries which produce a stream response. - `direction`: Determines the sort order of logs. Supported values are `forward` or `backward`. Defaults to `backward.` @@ -439,6 +440,7 @@ It accepts the following query parameters in the URL: - `start`: The start time for the query as a nanosecond Unix epoch. Defaults to 6 hours ago. - `end`: The end time for the query as a nanosecond Unix epoch. Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. In microservices mode, `/loki/api/v1/labels` is exposed by the querier. @@ -480,6 +482,7 @@ It accepts the following query parameters in the URL: - `start`: The start time for the query as a nanosecond Unix epoch. Defaults to 6 hours ago. - `end`: The end time for the query as a nanosecond Unix epoch. Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. In microservices mode, `/loki/api/v1/label//values` is exposed by the querier. @@ -731,6 +734,7 @@ URL query parameters: - `match[]=`: Repeated log stream selector argument that selects the streams to return. At least one `match[]` argument must be provided. - `start=`: Start timestamp. - `end=`: End timestamp. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. You can URL-encode these parameters directly in the request body by using the POST method and `Content-Type: application/x-www-form-urlencoded` header. This is useful when specifying a large or dynamic number of stream selectors that may breach server-side URL character limits. @@ -1217,6 +1221,7 @@ support the following values: - `limit`: The max number of entries to return - `start`: The start time for the query as a nanosecond Unix epoch. Defaults to one hour ago. - `end`: The end time for the query as a nanosecond Unix epoch. Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. - `direction`: Determines the sort order of logs. Supported values are `forward` or `backward`. Defaults to `backward.` - `regexp`: a regex to filter the returned results @@ -1284,6 +1289,7 @@ the URL: - `start`: The start time for the query as a nanosecond Unix epoch. Defaults to 6 hours ago. - `end`: The end time for the query as a nanosecond Unix epoch. Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. In microservices mode, `/api/prom/label//values` is exposed by the querier. @@ -1320,6 +1326,7 @@ accepts the following query parameters in the URL: - `start`: The start time for the query as a nanosecond Unix epoch. Defaults to 6 hours ago. - `end`: The end time for the query as a nanosecond Unix epoch. Defaults to now. +- `since`: A `duration` used to calculate `start` relative to `end`. If `end` is in the future, `start` is calculated as this duration before now. Any value specified for `start` supersedes this parameter. In microservices mode, `/api/prom/label` is exposed by the querier. diff --git a/pkg/loghttp/params.go b/pkg/loghttp/params.go index fe5da15a32..8d2739c025 100644 --- a/pkg/loghttp/params.go +++ b/pkg/loghttp/params.go @@ -48,14 +48,40 @@ func shards(r *http.Request) []string { func bounds(r *http.Request) (time.Time, time.Time, error) { now := time.Now() - start, err := parseTimestamp(r.Form.Get("start"), now.Add(-defaultSince)) + start := r.Form.Get("start") + end := r.Form.Get("end") + since := r.Form.Get("since") + return determineBounds(now, start, end, since) +} + +func determineBounds(now time.Time, startString, endString, sinceString string) (time.Time, time.Time, error) { + since := defaultSince + if sinceString != "" { + d, err := model.ParseDuration(sinceString) + if err != nil { + return time.Time{}, time.Time{}, errors.Wrap(err, "could not parse 'since' parameter") + } + since = time.Duration(d) + } + + end, err := parseTimestamp(endString, now) if err != nil { - return time.Time{}, time.Time{}, err + return time.Time{}, time.Time{}, errors.Wrap(err, "could not parse 'end' parameter") } - end, err := parseTimestamp(r.Form.Get("end"), now) + + // endOrNow is used to apply a default for the start time or an offset if 'since' is provided. + // we want to use the 'end' time so long as it's not in the future as this should provide + // a more intuitive experience when end time is in the future. + endOrNow := end + if end.After(now) { + endOrNow = now + } + + start, err := parseTimestamp(startString, endOrNow.Add(-since)) if err != nil { - return time.Time{}, time.Time{}, err + return time.Time{}, time.Time{}, errors.Wrap(err, "could not parse 'start' parameter") } + return start, end, nil } diff --git a/pkg/loghttp/params_test.go b/pkg/loghttp/params_test.go index b3f619b00c..873fdff36b 100644 --- a/pkg/loghttp/params_test.go +++ b/pkg/loghttp/params_test.go @@ -1,6 +1,7 @@ package loghttp import ( + "fmt" "net/http/httptest" "reflect" "testing" @@ -223,3 +224,144 @@ func Test_parseTimestamp(t *testing.T) { }) } } + +func Test_determineBounds(t *testing.T) { + type args struct { + now time.Time + startString string + endString string + sinceString string + } + tests := []struct { + name string + args args + start time.Time + end time.Time + wantErr assert.ErrorAssertionFunc + }{ + { + name: "no start, end, since", + args: args{ + now: time.Unix(3600, 0), + startString: "", + endString: "", + sinceString: "", + }, + start: time.Unix(0, 0), // Default start is one hour before 'now' if nothing is provided + end: time.Unix(3600, 0), // Default end is 'now' if nothing is provided + wantErr: assert.NoError, + }, + { + name: "no since or no start with end in the future", + args: args{ + now: time.Unix(3600, 0), + startString: "", + endString: "2022-12-18T00:00:00Z", + sinceString: "", + }, + start: time.Unix(0, 0), // Default should be one hour before now + end: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + wantErr: assert.NoError, + }, + { + name: "no since, valid start and end", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "2022-12-17T00:00:00Z", + endString: "2022-12-18T00:00:00Z", + sinceString: "", + }, + start: time.Date(2022, 12, 17, 0, 0, 0, 0, time.UTC), + end: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + wantErr: assert.NoError, + }, + { + name: "invalid end", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "2022-12-17T00:00:00Z", + endString: "WHAT TIME IS IT?", + sinceString: "", + }, + start: time.Time{}, + end: time.Time{}, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "could not parse 'end' parameter:", i...) + }, + }, + { + name: "invalid start", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "LET'S GOOO", + endString: "2022-12-18T00:00:00Z", + sinceString: "", + }, + start: time.Time{}, + end: time.Time{}, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "could not parse 'start' parameter:", i...) + }, + }, + { + name: "invalid since", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "2022-12-17T00:00:00Z", + endString: "2022-12-18T00:00:00Z", + sinceString: "HI!", + }, + start: time.Time{}, + end: time.Time{}, + wantErr: func(t assert.TestingT, err error, i ...interface{}) bool { + return assert.ErrorContains(t, err, "could not parse 'since' parameter:", i...) + }, + }, + { + name: "since 1h with no start or end", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "", + endString: "", + sinceString: "1h", + }, + start: time.Date(2022, 12, 17, 23, 0, 0, 0, time.UTC), + end: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + wantErr: assert.NoError, + }, + { + name: "since 1d with no start or end", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "", + endString: "", + sinceString: "1d", + }, + start: time.Date(2022, 12, 17, 0, 0, 0, 0, time.UTC), + end: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + wantErr: assert.NoError, + }, + { + name: "since 1h with no start and end time in the past", + args: args{ + now: time.Date(2022, 12, 18, 0, 0, 0, 0, time.UTC), + startString: "", + endString: "2022-12-17T00:00:00Z", + sinceString: "1h", + }, + start: time.Date(2022, 12, 16, 23, 0, 0, 0, time.UTC), // start should be calculated relative to end when end is specified + end: time.Date(2022, 12, 17, 0, 0, 0, 0, time.UTC), + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := determineBounds(tt.args.now, tt.args.startString, tt.args.endString, tt.args.sinceString) + if !tt.wantErr(t, err, fmt.Sprintf("determineBounds(%v, %v, %v, %v)", tt.args.now, tt.args.startString, tt.args.endString, tt.args.sinceString)) { + return + } + assert.Equalf(t, tt.start, got, "determineBounds(%v, %v, %v, %v)", tt.args.now, tt.args.startString, tt.args.endString, tt.args.sinceString) + assert.Equalf(t, tt.end, got1, "determineBounds(%v, %v, %v, %v)", tt.args.now, tt.args.startString, tt.args.endString, tt.args.sinceString) + }) + } +}