From 7e7436d482905a2cf8f5e1ad8bb588d062a8ee68 Mon Sep 17 00:00:00 2001 From: Mohamed-Amine Bouqsimi Date: Fri, 21 Oct 2022 10:13:41 +0200 Subject: [PATCH] operator: Expose and migrate query_timeout in limits config (#7471) --- operator/CHANGELOG.md | 1 + operator/apis/loki/v1/lokistack_types.go | 8 +++++++ .../loki-operator.clusterserviceversion.yaml | 8 +++++++ .../loki.grafana.com_lokistacks.yaml | 10 +++++++++ .../bases/loki.grafana.com_lokistacks.yaml | 10 +++++++++ .../loki-operator.clusterserviceversion.yaml | 8 +++++++ .../manifests/internal/config/build_test.go | 21 ++++++++++++------- .../internal/config/loki-config.yaml | 2 +- .../internal/config/loki-runtime-config.yaml | 3 +++ operator/internal/manifests/internal/sizes.go | 3 +++ 10 files changed, 66 insertions(+), 8 deletions(-) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 1dba1d9474..88627d13e8 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,5 +1,6 @@ ## Main +- [7471](https://github.com/grafana/loki/pull/7471) **aminesnow**: Expose and migrate query_timeout in limits config - [7437](https://github.com/grafana/loki/pull/7437) **aminesnow**: Fix Custom TLS profile setting for LokiStack on OpenShift - [7415](https://github.com/grafana/loki/pull/7415) **aminesnow**: Add alert relabel config - [7418](https://github.com/grafana/loki/pull/7418) **Red-GV**: Update golang to v1.19 and k8s dependencies to v0.25.2 diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 02e7ee6afe..da24185c4c 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -462,6 +462,14 @@ type QueryLimitSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:number",displayName="Max Query Series" MaxQuerySeries int32 `json:"maxQuerySeries,omitempty"` + + // Timeout when querying ingesters or storage during the execution of a query request. + // + // +optional + // +kubebuilder:validation:Optional + // +kubebuilder:default:="1m" + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Query Timeout" + QueryTimeout string `json:"queryTimeout,omitempty"` } // IngestionLimitSpec defines the limits applied at the ingestion path. diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index 701e2cd438..ba635e196b 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -337,6 +337,10 @@ spec: path: limits.global.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Timeout when querying ingesters or storage during the execution + of a query request. + displayName: Query Timeout + path: limits.global.queries.queryTimeout - description: Tenants defines the limits applied per tenant. displayName: Limits per Tenant path: limits.tenants @@ -400,6 +404,10 @@ spec: path: limits.tenants.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Timeout when querying ingesters or storage during the execution + of a query request. + displayName: Query Timeout + path: limits.tenants.queries.queryTimeout - description: ManagementState defines if the CR should be managed by the operator or not. Default is managed. displayName: Management State diff --git a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml index f17269b5d5..57e11f45b4 100644 --- a/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/manifests/loki.grafana.com_lokistacks.yaml @@ -123,6 +123,11 @@ spec: unique series that is returned by a metric query. format: int32 type: integer + queryTimeout: + default: 1m + description: Timeout when querying ingesters or storage + during the execution of a query request. + type: string type: object retention: description: Retention defines how long logs are kept in storage. @@ -231,6 +236,11 @@ spec: of unique series that is returned by a metric query. format: int32 type: integer + queryTimeout: + default: 1m + description: Timeout when querying ingesters or storage + during the execution of a query request. + type: string type: object retention: description: Retention defines how long logs are kept in diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 8eddc86436..400129cef7 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -106,6 +106,11 @@ spec: unique series that is returned by a metric query. format: int32 type: integer + queryTimeout: + default: 1m + description: Timeout when querying ingesters or storage + during the execution of a query request. + type: string type: object retention: description: Retention defines how long logs are kept in storage. @@ -214,6 +219,11 @@ spec: of unique series that is returned by a metric query. format: int32 type: integer + queryTimeout: + default: 1m + description: Timeout when querying ingesters or storage + during the execution of a query request. + type: string type: object retention: description: Retention defines how long logs are kept in diff --git a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml index d5f8a44272..918c12dc2a 100644 --- a/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/loki-operator.clusterserviceversion.yaml @@ -192,6 +192,10 @@ spec: path: limits.global.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Timeout when querying ingesters or storage during the execution + of a query request. + displayName: Query Timeout + path: limits.global.queries.queryTimeout - description: Tenants defines the limits applied per tenant. displayName: Limits per Tenant path: limits.tenants @@ -255,6 +259,10 @@ spec: path: limits.tenants.queries.maxQuerySeries x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Timeout when querying ingesters or storage during the execution + of a query request. + displayName: Query Timeout + path: limits.tenants.queries.queryTimeout - description: ManagementState defines if the CR should be managed by the operator or not. Default is managed. displayName: Management State diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 042fe7daa4..752c243ed9 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -101,6 +101,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -116,7 +117,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -184,6 +184,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -337,6 +338,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -352,7 +354,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -425,6 +426,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, Tenants: map[string]lokiv1.LimitsTemplateSpec{ @@ -664,6 +666,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -679,7 +682,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -801,6 +803,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -1001,6 +1004,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -1016,7 +1020,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -1138,6 +1141,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -1339,6 +1343,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -1354,7 +1359,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -1489,6 +1493,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -1715,6 +1720,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 1m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -1730,7 +1736,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -1808,6 +1813,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, Retention: &lokiv1.RetentionLimitSpec{ Days: 15, @@ -1996,6 +2002,7 @@ limits_config: per_stream_rate_limit: 3MB per_stream_rate_limit_burst: 15MB split_queries_by_interval: 30m + query_timeout: 2m memberlist: abort_if_cluster_join_fails: true bind_port: 7946 @@ -2011,7 +2018,6 @@ querier: extra_query_delay: 0s max_concurrent: 2 query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h query_range: align_queries_with_step: true @@ -2159,6 +2165,7 @@ overrides: MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "2m", }, }, }, diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index c5c72bb772..85f904883d 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -120,6 +120,7 @@ limits_config: max_query_series: {{ .Stack.Limits.Global.QueryLimits.MaxQuerySeries }} cardinality_limit: 100000 max_streams_matchers_per_query: 1000 + query_timeout: {{ .Stack.Limits.Global.QueryLimits.QueryTimeout }} {{- if .Retention.Enabled }}{{- with .Stack.Limits.Global.Retention }} retention_period: {{.Days}}d {{- with .Streams }} @@ -149,7 +150,6 @@ querier: timeout: 3m extra_query_delay: 0s query_ingesters_within: 3h - query_timeout: 1m tail_max_duration: 1h max_concurrent: {{ .MaxConcurrent.AvailableQuerierCPUCores }} query_range: diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index e330cc0419..de9dd519fe 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -35,6 +35,9 @@ overrides: {{ if $spec.QueryLimits.MaxQuerySeries }} max_query_series: {{ $spec.QueryLimits.MaxQuerySeries }} {{- end -}} + {{ if $spec.QueryLimits.QueryTimeout }} + query_timeout: {{ $spec.QueryLimits.QueryTimeout }} + {{- end -}} {{- end -}} {{- with $spec.Retention }} retention_period: {{ .Days }}d diff --git a/operator/internal/manifests/internal/sizes.go b/operator/internal/manifests/internal/sizes.go index fb06a0f7c3..468fac2e5d 100644 --- a/operator/internal/manifests/internal/sizes.go +++ b/operator/internal/manifests/internal/sizes.go @@ -223,6 +223,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -275,6 +276,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, }, @@ -327,6 +329,7 @@ var StackSizeTable = map[lokiv1.LokiStackSizeType]lokiv1.LokiStackSpec{ MaxEntriesLimitPerQuery: 5000, MaxChunksPerQuery: 2000000, MaxQuerySeries: 500, + QueryTimeout: "1m", }, }, },