From 999259a4d335a0238a31ff8325490f12321c058e Mon Sep 17 00:00:00 2001 From: manohar-koukuntla <108517792+manohar-koukuntla@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:17:33 +0200 Subject: [PATCH] jsonnet: Add TSDB shipper configuration flag (#9790) This commit adds the TSDB equivalent of `use_boltdb_shipper` flag to be able to configure `tsdb_shipper` section. Fixes #9789 --- CHANGELOG.md | 1 + docs/sources/installation/tanka.md | 2 +- production/ksonnet/loki/config.libsonnet | 3 -- .../ksonnet/loki/index-gateway.libsonnet | 5 +++ production/ksonnet/loki/loki.libsonnet | 4 +- ...db_shipper.libsonnet => shipper.libsonnet} | 38 ++++++++++++------- 6 files changed, 34 insertions(+), 19 deletions(-) rename production/ksonnet/loki/{boltdb_shipper.libsonnet => shipper.libsonnet} (59%) diff --git a/CHANGELOG.md b/CHANGELOG.md index dddf83048b..a45c2c516b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,7 @@ #### Jsonnet +* [9790](https://github.com/grafana/loki/pull/9790) **manohar-koukuntla**: Add TSDB equivalent of `use_boltdb_shipper` flag to be able to configure `tsdb_shipper` section. * [8855](https://github.com/grafana/loki/pull/8855) **JoaoBraveCoding**: Add gRPC port to loki compactor mixin * [8880](https://github.com/grafana/loki/pull/8880) **JoaoBraveCoding**: Normalize headless service name for query-frontend/scheduler diff --git a/docs/sources/installation/tanka.md b/docs/sources/installation/tanka.md index a9419d0d2f..4f84e4d1d9 100644 --- a/docs/sources/installation/tanka.md +++ b/docs/sources/installation/tanka.md @@ -45,7 +45,7 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var - Update the `username`, `password`, and the relevant `htpasswd` variable values. - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki/latest/configuration/#storage_config) for more configuration details. - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup. -- Update the value of `boltdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. +- Update the value of `boltdb_shipper_shared_store` or `tsdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path. - Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range. diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet index d4b0e322f5..bae016daf8 100644 --- a/production/ksonnet/loki/config.libsonnet +++ b/production/ksonnet/loki/config.libsonnet @@ -12,9 +12,6 @@ grpc_server_max_msg_size: 100 << 20, // 100MB - // flag for tuning things when boltdb-shipper is current or upcoming index type. - using_boltdb_shipper: true, - wal_enabled: true, query_scheduler_enabled: false, overrides_exporter_enabled: false, diff --git a/production/ksonnet/loki/index-gateway.libsonnet b/production/ksonnet/loki/index-gateway.libsonnet index 513c76c031..de3e001fe2 100644 --- a/production/ksonnet/loki/index-gateway.libsonnet +++ b/production/ksonnet/loki/index-gateway.libsonnet @@ -11,6 +11,11 @@ server_address: 'dns:///index-gateway.%s.svc.cluster.local:9095' % $._config.namespace, }, }, + tsdb_shipper+: { + index_gateway_client+: { + server_address: 'dns:///index-gateway.%s.svc.cluster.local:9095' % $._config.namespace, + }, + }, } else {}, }, }, diff --git a/production/ksonnet/loki/loki.libsonnet b/production/ksonnet/loki/loki.libsonnet index 7d017804e2..3179248e22 100644 --- a/production/ksonnet/loki/loki.libsonnet +++ b/production/ksonnet/loki/loki.libsonnet @@ -27,8 +27,8 @@ // Index Gateway support (import 'index-gateway.libsonnet') + -// BoltDB Shipper support. Anything that modifies the compactor must be imported after this. -(import 'boltdb_shipper.libsonnet') + +// BoltDB and TSDB Shipper support. Anything that modifies the compactor must be imported after this. +(import 'shipper.libsonnet') + // Multi-zone ingester related config (import 'multi-zone.libsonnet') + diff --git a/production/ksonnet/loki/boltdb_shipper.libsonnet b/production/ksonnet/loki/shipper.libsonnet similarity index 59% rename from production/ksonnet/loki/boltdb_shipper.libsonnet rename to production/ksonnet/loki/shipper.libsonnet index 4c0ef90ca5..c7355f0299 100644 --- a/production/ksonnet/loki/boltdb_shipper.libsonnet +++ b/production/ksonnet/loki/shipper.libsonnet @@ -8,48 +8,60 @@ local containerPort = k.core.v1.containerPort, _config+:: { + // flag for tuning things when boltdb-shipper is current or upcoming index type. + using_boltdb_shipper: true, + using_tsdb_shipper: false, + using_shipper_store: $._config.using_boltdb_shipper || $._config.using_tsdb_shipper, + + boltdb_shipper_shared_store: error 'must define boltdb_shipper_shared_store when using_boltdb_shipper=true. If this is not intentional, consider disabling it. shared_store is a backend key from the storage_config, such as (gcs) or (s3)', + tsdb_shipper_shared_store: error 'must define tsdb_shipper_shared_store when using_tsdb_shipper=true. If this is not intentional, consider disabling it. shared_store is a backend key from the storage_config, such as (gcs) or (s3)', + // run ingesters and queriers as statefulsets when using boltdb-shipper to avoid using node disk for storing the index. - stateful_ingesters: if self.using_boltdb_shipper then true else super.stateful_ingesters, - stateful_queriers: if self.using_boltdb_shipper && !self.use_index_gateway then true else super.stateful_queriers, + stateful_ingesters: if self.using_shipper_store then true else super.stateful_ingesters, + stateful_queriers: if self.using_shipper_store && !self.use_index_gateway then true else super.stateful_queriers, - boltdb_shipper_shared_store: error 'must define boltdb_shipper_shared_store when using_boltdb_shipper=true. If this is not intentional, consider disabling it. boltdb_shipper_shared_store is a backend key from the storage_config, such as (gcs) or (s3)', compactor_pvc_size: '10Gi', compactor_pvc_class: 'fast', - index_period_hours: if self.using_boltdb_shipper then 24 else super.index_period_hours, - loki+: if self.using_boltdb_shipper then { + index_period_hours: if self.using_shipper_store then 24 else super.index_period_hours, + loki+: if self.using_shipper_store then { storage_config+: { boltdb_shipper+: { shared_store: $._config.boltdb_shipper_shared_store, active_index_directory: '/data/index', cache_location: '/data/boltdb-cache', }, + tsdb_shipper+: { + shared_store: $._config.tsdb_shipper_shared_store, + active_index_directory: '/data/tsdb-index', + cache_location: '/data/tsdb-cache', + }, }, compactor+: { working_directory: '/data/compactor', - shared_store: $._config.boltdb_shipper_shared_store, + shared_store: if self.using_boltdb_shipper then self.boltdb_shipper_shared_store else self.tsdb_shipper_shared_store, }, } else {}, }, - // we don't dedupe index writes when using boltdb-shipper so don't deploy a cache for it. - memcached_index_writes: if $._config.using_boltdb_shipper then {} else + // we don't dedupe index writes when using boltdb-shipper or tsdb-shipper so don't deploy a cache for it. + memcached_index_writes: if $._config.using_shipper_store then {} else if 'memcached_index_writes' in super then super.memcached_index_writes else {}, // Use PVC for compactor instead of node disk. - compactor_data_pvc:: if $._config.using_boltdb_shipper then + compactor_data_pvc:: if $._config.using_shipper_store then pvc.new('compactor-data') + pvc.mixin.spec.resources.withRequests({ storage: $._config.compactor_pvc_size }) + pvc.mixin.spec.withAccessModes(['ReadWriteOnce']) + pvc.mixin.spec.withStorageClassName($._config.compactor_pvc_class) else {}, - compactor_args:: if $._config.using_boltdb_shipper then $._config.commonArgs { + compactor_args:: if $._config.using_shipper_store then $._config.commonArgs { target: 'compactor', } else {}, compactor_ports:: $.util.defaultPorts, - compactor_container:: if $._config.using_boltdb_shipper then + compactor_container:: if $._config.using_shipper_store then container.new('compactor', $._images.compactor) + container.withPorts($.compactor_ports) + container.withArgsMixin(k.util.mapToFlags($.compactor_args)) + @@ -61,7 +73,7 @@ container.withEnvMixin($._config.commonEnvs) else {}, - compactor_statefulset: if $._config.using_boltdb_shipper then + compactor_statefulset: if $._config.using_shipper_store then statefulSet.new('compactor', 1, [$.compactor_container], $.compactor_data_pvc) + statefulSet.mixin.spec.withServiceName('compactor') + $.config_hash_mixin + @@ -71,7 +83,7 @@ statefulSet.mixin.spec.template.spec.securityContext.withFsGroup(10001) // 10001 is the group ID assigned to Loki in the Dockerfile else {}, - compactor_service: if $._config.using_boltdb_shipper then + compactor_service: if $._config.using_shipper_store then k.util.serviceFor($.compactor_statefulset, $._config.service_ignored_labels) else {}, }