Log query string on retry alongside the error (#7834)

**What this PR does / why we need it**:

For better observability of query retries.

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

pkg/querier/queryrange/queryrangebase/retry.go

@@ -70,7 +70,7 @@ func (r retry) Do(ctx context.Context, req Request) (Response, error) {
 		httpResp, ok := httpgrpc.HTTPResponseFromError(err)
 		if !ok || httpResp.Code/100 == 5 {
 			lastErr = err
-			level.Error(util_log.WithContext(ctx, r.log)).Log("msg", "error processing request", "try", tries, "err", err)
+			level.Error(util_log.WithContext(ctx, r.log)).Log("msg", "error processing request", "try", tries, "query", req.GetQuery(), "err", err)
 			continue
 		}
 

pkg/querier/queryrange/queryrangebase/retry_test.go

@@ -60,7 +60,10 @@ func TestRetry(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			try.Store(0)
 			h := NewRetryMiddleware(log.NewNopLogger(), 5, nil).Wrap(tc.handler)
-			resp, err := h.Do(context.Background(), nil)
+			req := &PrometheusRequest{
+				Query: `{env="test"} |= "error"`,
+			}
+			resp, err := h.Do(context.Background(), req)
 			require.Equal(t, tc.err, err)
 			require.Equal(t, tc.resp, resp)
 		})
@@ -68,6 +71,10 @@ func TestRetry(t *testing.T) {
 }
 
 func Test_RetryMiddlewareCancel(t *testing.T) {
+	req := &PrometheusRequest{
+		Query: `{env="test"} |= "error"`,
+	}
+
 	var try atomic.Int32
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
@@ -76,7 +83,7 @@ func Test_RetryMiddlewareCancel(t *testing.T) {
 			try.Inc()
 			return nil, ctx.Err()
 		}),
-	).Do(ctx, nil)
+	).Do(ctx, req)
 	require.Equal(t, int32(0), try.Load())
 	require.Equal(t, ctx.Err(), err)
 
@@ -87,7 +94,7 @@ func Test_RetryMiddlewareCancel(t *testing.T) {
 			cancel()
 			return nil, errors.New("failed")
 		}),
-	).Do(ctx, nil)
+	).Do(ctx, req)
 	require.Equal(t, int32(1), try.Load())
 	require.Equal(t, ctx.Err(), err)
 }