operator: Expose and migrate query_timeout in limits config (#7471)

3 years ago · 7e7436d482
parent 28e192f22e
commit 7e7436d482
10 changed files with 66 additions and 8 deletions
--- a/operator/CHANGELOG.md
+++ b/operator/CHANGELOG.md
@ -1,5 +1,6 @@
 ## Main

+- [7471](https://github.com/grafana/loki/pull/7471) **aminesnow**: Expose and migrate query_timeout in limits config
 - [7437](https://github.com/grafana/loki/pull/7437) **aminesnow**: Fix Custom TLS profile setting for LokiStack on OpenShift
 - [7415](https://github.com/grafana/loki/pull/7415) **aminesnow**: Add alert relabel config
 - [7418](https://github.com/grafana/loki/pull/7418) **Red-GV**: Update golang to v1.19 and k8s dependencies to v0.25.2
--- a/operator/apis/loki/v1/lokistack_types.go
+++ b/operator/apis/loki/v1/lokistack_types.go
@ -462,6 +462,14 @@ type QueryLimitSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Query Series"
 	MaxQuerySeries int32 `json:"maxQuerySeries,omitempty"`
+
+	// Timeout when querying ingesters or storage during the execution of a query request.
+	//
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="1m"
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Query Timeout"
+	QueryTimeout string `json:"queryTimeout,omitempty"`
 }

 // IngestionLimitSpec defines the limits applied at the ingestion path.
--- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
+++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
@ -337,6 +337,10 @@ spec:
        path: limits.global.queries.maxQuerySeries
        x-descriptors:
        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Timeout when querying ingesters or storage during the execution
+          of a query request.
+        displayName: Query Timeout
+        path: limits.global.queries.queryTimeout
      - description: Tenants defines the limits applied per tenant.
        displayName: Limits per Tenant
        path: limits.tenants
@ -400,6 +404,10 @@ spec:
        path: limits.tenants.queries.maxQuerySeries
        x-descriptors:
        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Timeout when querying ingesters or storage during the execution
+          of a query request.
+        displayName: Query Timeout
+        path: limits.tenants.queries.queryTimeout
      - description: ManagementState defines if the CR should be managed by the operator
          or not. Default is managed.
        displayName: Management State
--- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
+++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml
@ -123,6 +123,11 @@ spec:
                              unique series that is returned by a metric query.
                            format: int32
                            type: integer
+                          queryTimeout:
+                            default: 1m
+                            description: Timeout when querying ingesters or storage
+                              during the execution of a query request.
+                            type: string
                        type: object
                      retention:
                        description: Retention defines how long logs are kept in storage.
@ -231,6 +236,11 @@ spec:
                                of unique series that is returned by a metric query.
                              format: int32
                              type: integer
+                            queryTimeout:
+                              default: 1m
+                              description: Timeout when querying ingesters or storage
+                                during the execution of a query request.
+                              type: string
                          type: object
                        retention:
                          description: Retention defines how long logs are kept in
--- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
+++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml
@ -106,6 +106,11 @@ spec:
                              unique series that is returned by a metric query.
                            format: int32
                            type: integer
+                          queryTimeout:
+                            default: 1m
+                            description: Timeout when querying ingesters or storage
+                              during the execution of a query request.
+                            type: string
                        type: object
                      retention:
                        description: Retention defines how long logs are kept in storage.
@ -214,6 +219,11 @@ spec:
                                of unique series that is returned by a metric query.
                              format: int32
                              type: integer
+                            queryTimeout:
+                              default: 1m
+                              description: Timeout when querying ingesters or storage
+                                during the execution of a query request.
+                              type: string
                          type: object
                        retention:
                          description: Retention defines how long logs are kept in
--- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
+++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml
@ -192,6 +192,10 @@ spec:
        path: limits.global.queries.maxQuerySeries
        x-descriptors:
        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Timeout when querying ingesters or storage during the execution
+          of a query request.
+        displayName: Query Timeout
+        path: limits.global.queries.queryTimeout
      - description: Tenants defines the limits applied per tenant.
        displayName: Limits per Tenant
        path: limits.tenants
@ -255,6 +259,10 @@ spec:
        path: limits.tenants.queries.maxQuerySeries
        x-descriptors:
        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Timeout when querying ingesters or storage during the execution
+          of a query request.
+        displayName: Query Timeout
+        path: limits.tenants.queries.queryTimeout
      - description: ManagementState defines if the CR should be managed by the operator
          or not. Default is managed.
        displayName: Management State
--- a/operator/internal/manifests/internal/config/build_test.go
+++ b/operator/internal/manifests/internal/config/build_test.go
@ -101,6 +101,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -116,7 +117,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -184,6 +184,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 				},
 			},
@ -337,6 +338,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -352,7 +354,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -425,6 +426,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 				},
 				Tenants: map[string]lokiv1.LimitsTemplateSpec{
@ -664,6 +666,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -679,7 +682,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -801,6 +803,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 				},
 			},
@ -1001,6 +1004,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -1016,7 +1020,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -1138,6 +1141,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 				},
 			},
@ -1339,6 +1343,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -1354,7 +1359,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -1489,6 +1493,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 				},
 			},
@ -1715,6 +1720,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 1m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -1730,7 +1736,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -1808,6 +1813,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "1m",
 					},
 					Retention: &lokiv1.RetentionLimitSpec{
 						Days: 15,
@ -1996,6 +2002,7 @@ limits_config:
  per_stream_rate_limit: 3MB
  per_stream_rate_limit_burst: 15MB
  split_queries_by_interval: 30m
+  query_timeout: 2m
 memberlist:
  abort_if_cluster_join_fails: true
  bind_port: 7946
@ -2011,7 +2018,6 @@ querier:
  extra_query_delay: 0s
  max_concurrent: 2
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
 query_range:
  align_queries_with_step: true
@ -2159,6 +2165,7 @@ overrides:
 						MaxEntriesLimitPerQuery: 5000,
 						MaxChunksPerQuery:       2000000,
 						MaxQuerySeries:          500,
+						QueryTimeout:            "2m",
 					},
 				},
 			},
--- a/operator/internal/manifests/internal/config/loki-config.yaml
+++ b/operator/internal/manifests/internal/config/loki-config.yaml
@ -120,6 +120,7 @@ limits_config:
  max_query_series: {{ .Stack.Limits.Global.QueryLimits.MaxQuerySeries }}
  cardinality_limit: 100000
  max_streams_matchers_per_query: 1000
+  query_timeout: {{ .Stack.Limits.Global.QueryLimits.QueryTimeout }}
 {{- if .Retention.Enabled }}{{- with .Stack.Limits.Global.Retention }}
  retention_period: {{.Days}}d
 {{- with .Streams }}
@ -149,7 +150,6 @@ querier:
    timeout: 3m
  extra_query_delay: 0s
  query_ingesters_within: 3h
-  query_timeout: 1m
  tail_max_duration: 1h
  max_concurrent: {{ .MaxConcurrent.AvailableQuerierCPUCores }}
 query_range:
--- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml
+++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml
@ -35,6 +35,9 @@ overrides:
    {{ if $spec.QueryLimits.MaxQuerySeries }}
    max_query_series: {{ $spec.QueryLimits.MaxQuerySeries }}
    {{- end -}}
+    {{ if $spec.QueryLimits.QueryTimeout }}
+    query_timeout: {{ $spec.QueryLimits.QueryTimeout }}
+    {{- end -}}
  {{- end -}}
  {{- with $spec.Retention }}
    retention_period: {{ .Days }}d
--- a/operator/internal/manifests/internal/sizes.go
+++ b/operator/internal/manifests/internal/sizes.go
@ -223,6 +223,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
 					MaxEntriesLimitPerQuery: 5000,
 					MaxChunksPerQuery:       2000000,
 					MaxQuerySeries:          500,
+					QueryTimeout:            "1m",
 				},
 			},
 		},
@ -275,6 +276,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
 					MaxEntriesLimitPerQuery: 5000,
 					MaxChunksPerQuery:       2000000,
 					MaxQuerySeries:          500,
+					QueryTimeout:            "1m",
 				},
 			},
 		},
@ -327,6 +329,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{
 					MaxEntriesLimitPerQuery: 5000,
 					MaxChunksPerQuery:       2000000,
 					MaxQuerySeries:          500,
+					QueryTimeout:            "1m",
 				},
 			},
 		},