Loki: Improve error messages on query timeout or cancel (#2453)

* Instead of returning "context deadline exceeded" on timeout and "context cancelled" when the query is cancelled, return a more descriptive error message.

* fixing tests

pkg/util/server/error.go

@@ -14,15 +14,20 @@ import (
 // StatusClientClosedRequest is the status code for when a client request cancellation of an http request
 const StatusClientClosedRequest = 499
 
+const (
+	ErrClientCanceled   = "The request was cancelled by the client."
+	ErrDeadlineExceeded = "Request timed out, decrease the duration of the request or add more label matchers (prefer exact match over regex match) to reduce the amount of data processed."
+)
+
 // WriteError write a go error with the correct status code.
 func WriteError(err error, w http.ResponseWriter) {
 	var queryErr chunk.QueryError
 
 	switch {
 	case errors.Is(err, context.Canceled):
-		http.Error(w, err.Error(), StatusClientClosedRequest)
+		http.Error(w, ErrClientCanceled, StatusClientClosedRequest)
 	case errors.Is(err, context.DeadlineExceeded):
-		http.Error(w, err.Error(), http.StatusGatewayTimeout)
+		http.Error(w, ErrDeadlineExceeded, http.StatusGatewayTimeout)
 	case errors.As(err, &queryErr):
 		http.Error(w, err.Error(), http.StatusBadRequest)
 	case logql.IsParseError(err):

pkg/util/server/error_test.go

@@ -24,9 +24,8 @@ func Test_writeError(t *testing.T) {
 		msg            string
 		expectedStatus int
 	}{
-		{"cancelled", context.Canceled, context.Canceled.Error(), StatusClientClosedRequest},
-		{"wrapped cancelled", fmt.Errorf("some context here: %w", context.Canceled), "some context here: " + context.Canceled.Error(), StatusClientClosedRequest},
-		{"deadline", context.DeadlineExceeded, context.DeadlineExceeded.Error(), http.StatusGatewayTimeout},
+		{"cancelled", context.Canceled, ErrClientCanceled, StatusClientClosedRequest},
+		{"deadline", context.DeadlineExceeded, ErrDeadlineExceeded, http.StatusGatewayTimeout},
 		{"parse error", logql.ParseError{}, "parse error : ", http.StatusBadRequest},
 		{"httpgrpc", httpgrpc.Errorf(http.StatusBadRequest, errors.New("foo").Error()), "foo", http.StatusBadRequest},
 		{"internal", errors.New("foo"), "foo", http.StatusInternalServerError},