jsonnet: Add TSDB shipper configuration flag (#9790)

This commit adds the TSDB equivalent of `use_boltdb_shipper` flag to be able to configure `tsdb_shipper` section.

Fixes #9789

CHANGELOG.md

@@ -104,6 +104,7 @@
 
 #### Jsonnet
 
+* [9790](https://github.com/grafana/loki/pull/9790) **manohar-koukuntla**: Add TSDB equivalent of `use_boltdb_shipper` flag to be able to configure `tsdb_shipper` section.
 * [8855](https://github.com/grafana/loki/pull/8855) **JoaoBraveCoding**: Add gRPC port to loki compactor mixin
 * [8880](https://github.com/grafana/loki/pull/8880) **JoaoBraveCoding**: Normalize headless service name for query-frontend/scheduler
 

docs/sources/installation/tanka.md

@@ -45,7 +45,7 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var
 - Update the `username`, `password`, and the relevant `htpasswd` variable values.
 - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki/latest/configuration/#storage_config) for more configuration details.
 - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup.
-- Update the value of `boltdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. 
+- Update the value of `boltdb_shipper_shared_store` or `tsdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. 
 - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path.
 - Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range.
 

production/ksonnet/loki/config.libsonnet

@@ -12,9 +12,6 @@
 
     grpc_server_max_msg_size: 100 << 20,  // 100MB
 
-    // flag for tuning things when boltdb-shipper is current or upcoming index type.
-    using_boltdb_shipper: true,
-
     wal_enabled: true,
     query_scheduler_enabled: false,
     overrides_exporter_enabled: false,

production/ksonnet/loki/index-gateway.libsonnet

@@ -11,6 +11,11 @@
             server_address: 'dns:///index-gateway.%s.svc.cluster.local:9095' % $._config.namespace,
           },
         },
+        tsdb_shipper+: {
+          index_gateway_client+: {
+            server_address: 'dns:///index-gateway.%s.svc.cluster.local:9095' % $._config.namespace,
+          },
+        },
       } else {},
     },
   },

production/ksonnet/loki/loki.libsonnet

@@ -27,8 +27,8 @@
 // Index Gateway support
 (import 'index-gateway.libsonnet') +
 
-// BoltDB Shipper support. Anything that modifies the compactor must be imported after this.
-(import 'boltdb_shipper.libsonnet') +
+// BoltDB and TSDB Shipper support. Anything that modifies the compactor must be imported after this.
+(import 'shipper.libsonnet') +
 
 // Multi-zone ingester related config
 (import 'multi-zone.libsonnet') +

production/ksonnet/loki/shipper.libsonnet

@@ -8,48 +8,60 @@
   local containerPort = k.core.v1.containerPort,
 
   _config+:: {
+    // flag for tuning things when boltdb-shipper is current or upcoming index type.
+    using_boltdb_shipper: true,
+    using_tsdb_shipper: false,
+    using_shipper_store: $._config.using_boltdb_shipper || $._config.using_tsdb_shipper,
+
+    boltdb_shipper_shared_store: error 'must define boltdb_shipper_shared_store when using_boltdb_shipper=true. If this is not intentional, consider disabling it. shared_store is a backend key from the storage_config, such as (gcs) or (s3)',
+    tsdb_shipper_shared_store: error 'must define tsdb_shipper_shared_store when using_tsdb_shipper=true. If this is not intentional, consider disabling it. shared_store is a backend key from the storage_config, such as (gcs) or (s3)',
+
     // run ingesters and queriers as statefulsets when using boltdb-shipper to avoid using node disk for storing the index.
-    stateful_ingesters: if self.using_boltdb_shipper then true else super.stateful_ingesters,
-    stateful_queriers: if self.using_boltdb_shipper && !self.use_index_gateway then true else super.stateful_queriers,
+    stateful_ingesters: if self.using_shipper_store then true else super.stateful_ingesters,
+    stateful_queriers: if self.using_shipper_store && !self.use_index_gateway then true else super.stateful_queriers,
 
-    boltdb_shipper_shared_store: error 'must define boltdb_shipper_shared_store when using_boltdb_shipper=true. If this is not intentional, consider disabling it. boltdb_shipper_shared_store is a backend key from the storage_config, such as (gcs) or (s3)',
     compactor_pvc_size: '10Gi',
     compactor_pvc_class: 'fast',
-    index_period_hours: if self.using_boltdb_shipper then 24 else super.index_period_hours,
-    loki+: if self.using_boltdb_shipper then {
+    index_period_hours: if self.using_shipper_store then 24 else super.index_period_hours,
+    loki+: if self.using_shipper_store then {
       storage_config+: {
         boltdb_shipper+: {
           shared_store: $._config.boltdb_shipper_shared_store,
           active_index_directory: '/data/index',
           cache_location: '/data/boltdb-cache',
         },
+        tsdb_shipper+: {
+          shared_store: $._config.tsdb_shipper_shared_store,
+          active_index_directory: '/data/tsdb-index',
+          cache_location: '/data/tsdb-cache',
+        },
       },
       compactor+: {
         working_directory: '/data/compactor',
-        shared_store: $._config.boltdb_shipper_shared_store,
+        shared_store: if self.using_boltdb_shipper then self.boltdb_shipper_shared_store else self.tsdb_shipper_shared_store,
       },
     } else {},
   },
 
-  // we don't dedupe index writes when using boltdb-shipper so don't deploy a cache for it.
-  memcached_index_writes: if $._config.using_boltdb_shipper then {} else
+  // we don't dedupe index writes when using boltdb-shipper or tsdb-shipper so don't deploy a cache for it.
+  memcached_index_writes: if $._config.using_shipper_store then {} else
     if 'memcached_index_writes' in super then super.memcached_index_writes else {},
 
   // Use PVC for compactor instead of node disk.
-  compactor_data_pvc:: if $._config.using_boltdb_shipper then
+  compactor_data_pvc:: if $._config.using_shipper_store then
     pvc.new('compactor-data') +
     pvc.mixin.spec.resources.withRequests({ storage: $._config.compactor_pvc_size }) +
     pvc.mixin.spec.withAccessModes(['ReadWriteOnce']) +
     pvc.mixin.spec.withStorageClassName($._config.compactor_pvc_class)
   else {},
 
-  compactor_args:: if $._config.using_boltdb_shipper then $._config.commonArgs {
+  compactor_args:: if $._config.using_shipper_store then $._config.commonArgs {
     target: 'compactor',
   } else {},
 
   compactor_ports:: $.util.defaultPorts,
 
-  compactor_container:: if $._config.using_boltdb_shipper then
+  compactor_container:: if $._config.using_shipper_store then
     container.new('compactor', $._images.compactor) +
     container.withPorts($.compactor_ports) +
     container.withArgsMixin(k.util.mapToFlags($.compactor_args)) +
@@ -61,7 +73,7 @@
     container.withEnvMixin($._config.commonEnvs)
   else {},
 
-  compactor_statefulset: if $._config.using_boltdb_shipper then
+  compactor_statefulset: if $._config.using_shipper_store then
     statefulSet.new('compactor', 1, [$.compactor_container], $.compactor_data_pvc) +
     statefulSet.mixin.spec.withServiceName('compactor') +
     $.config_hash_mixin +
@@ -71,7 +83,7 @@
     statefulSet.mixin.spec.template.spec.securityContext.withFsGroup(10001)  // 10001 is the group ID assigned to Loki in the Dockerfile
   else {},
 
-  compactor_service: if $._config.using_boltdb_shipper then
+  compactor_service: if $._config.using_shipper_store then
     k.util.serviceFor($.compactor_statefulset, $._config.service_ignored_labels)
   else {},
 }