Fetch all entries with logcli if limit==0. (#8537)

CHANGELOG.md

@@ -73,6 +73,7 @@
 ##### Enhancement
 
 * [8413](https://github.com/grafana/loki/pull/8413) **chaudum**: Try to load tenant-specific `schemaconfig-{orgID}.yaml` when using `--remote-schema` argument and fallback to global `schemaconfig.yaml`.
+* [8537](https://github.com/grafana/loki/pull/8537) **jeschkies**: Allow fetching all entries with `--limit=0`.
 
 #### Fluent Bit
 

cmd/logcli/main.go

@@ -170,7 +170,7 @@ func main() {
 		}
 
 		// `--limit` doesn't make sense when using `--stdin` flag.
-		rangeQuery.Limit = math.MaxInt // TODO(kavi): is it a good idea?
+		rangeQuery.Limit = 0
 	}
 
 	switch cmd {
@@ -362,7 +362,7 @@ func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
 		return nil
 	})
 
-	cmd.Flag("limit", "Limit on number of entries to print.").Default("30").IntVar(&q.Limit)
+	cmd.Flag("limit", "Limit on number of entries to print. Setting it to 0 will fetch all entries.").Default("30").IntVar(&q.Limit)
 	if instant {
 		cmd.Arg("query", "eg 'rate({foo=\"bar\"} |~ \".*error.*\" [5m])'").Required().StringVar(&q.QueryString)
 		cmd.Flag("now", "Time at which to execute the instant query.").StringVar(&now)

pkg/logcli/query/query.go

@@ -92,7 +92,9 @@ func (q *Query) DoQuery(c client.Client, out output.LogOutput, statistics bool)
 		}
 		_, _ = q.printResult(resp.Data.Result, out, nil)
 	} else {
-		if q.Limit < q.BatchSize {
+		unlimited := q.Limit == 0
+
+		if q.Limit < q.BatchSize && !unlimited {
 			q.BatchSize = q.Limit
 		}
 		resultLength := 0
@@ -100,11 +102,12 @@ func (q *Query) DoQuery(c client.Client, out output.LogOutput, statistics bool)
 		start := q.Start
 		end := q.End
 		var lastEntry []*loghttp.Entry
-		for total < q.Limit {
+		for total < q.Limit || unlimited {
 			bs := q.BatchSize
 			// We want to truncate the batch size if the remaining number
 			// of items needed to reach the limit is less than the batch size
-			if q.Limit-total < q.BatchSize {
+			// unless the query has no limit, ie limit==0.
+			if q.Limit-total < q.BatchSize && !unlimited {
 				// Truncated batchsize is q.Limit - total, however we add to this
 				// the length of the overlap from the last query to make sure we get the
 				// correct amount of new logs knowing there will be some overlapping logs returned.

pkg/logcli/query/query_test.go

@@ -460,6 +460,46 @@ func Test_batch(t *testing.T) {
 				"line10", "line9", "line8", "line7", "line6b", "line6a", "line6", "line5", "line4", "line3", "line2",
 			},
 		},
+		{
+			name: "single stream backward batch identical timestamps without limit",
+			streams: []logproto.Stream{
+				{
+					Labels: "{test=\"simple\"}",
+					Entries: []logproto.Entry{
+						{Timestamp: time.Unix(1, 0), Line: "line1"},
+						{Timestamp: time.Unix(2, 0), Line: "line2"},
+						{Timestamp: time.Unix(3, 0), Line: "line3"},
+						{Timestamp: time.Unix(4, 0), Line: "line4"},
+						{Timestamp: time.Unix(5, 0), Line: "line5"},
+						{Timestamp: time.Unix(6, 0), Line: "line6"},
+						{Timestamp: time.Unix(6, 0), Line: "line6a"},
+						{Timestamp: time.Unix(6, 0), Line: "line6b"},
+						{Timestamp: time.Unix(7, 0), Line: "line7"},
+						{Timestamp: time.Unix(8, 0), Line: "line8"},
+						{Timestamp: time.Unix(9, 0), Line: "line9"},
+						{Timestamp: time.Unix(10, 0), Line: "line10"},
+					},
+				},
+			},
+			start:        time.Unix(1, 0),
+			end:          time.Unix(11, 0),
+			limit:        0,
+			batch:        4,
+			labelMatcher: "{test=\"simple\"}",
+			forward:      false,
+			// Our batchsize is 2 but each query will also return the overlapping last element from the
+			// previous batch, as such we only get one item per call so we make a lot of calls
+			// Call one:   line10 line9 line8 line7
+			// Call two:   line7 line6b line6a line6
+			// Call three: line6b line6a line6 line5
+			// Call four:  line5 line5 line3 line2
+			// Call five:  line1
+			// Call six:   -
+			expectedCalls: 6,
+			expected: []string{
+				"line10", "line9", "line8", "line7", "line6b", "line6a", "line6", "line5", "line4", "line3", "line2", "line1",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {