Introduce an "enable_groupcache" config to control initialization better (#6673)

* Introduce an "enable_groupcache" config to control initialization better

We initialize groupcache as a module, which happens after the config is parsed. A function named applyFIFOCacheConfig is called to enable fifocache if no cache is configured, and at the time of its calling the modules have not been initialized. This new "enable_groupcache" value is set if groupcache is configured in common, or this value is set manually. With this setting, we can prevent automatic fifocache settings.

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Add tests

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Simplifying condition

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

pkg/loki/config_wrapper.go

@@ -85,6 +85,8 @@ func (c *ConfigWrapper) ApplyDynamicConfig() cfg.Source {
 
 		applyInstanceConfigs(r, &defaults)
 
+		applyCommonCacheConfigs(r, &defaults)
+
 		applyCommonReplicationFactor(r, &defaults)
 
 		applyDynamicRingConfigs(r, &defaults)
@@ -166,6 +168,17 @@ func applyInstanceConfigs(r, defaults *ConfigWrapper) {
 	}
 }
 
+// applyCommonCacheConfigs applies to Loki components the cache-related configurations under the common config section
+// NOTE: only used for GroupCache at the moment
+// TODO: apply to other caches as well
+func applyCommonCacheConfigs(r, _ *ConfigWrapper) {
+	if r.Config.Common.GroupCacheConfig.Enabled {
+		r.Config.ChunkStoreConfig.ChunkCacheConfig.EnableGroupCache = true
+		r.Config.QueryRange.ResultsCacheConfig.CacheConfig.EnableGroupCache = true
+		r.Config.StorageConfig.IndexQueriesCacheConfig.EnableGroupCache = true
+	}
+}
+
 // applyCommonReplicationFactor apply the common replication factor to the Index Gateway ring.
 func applyCommonReplicationFactor(r, defaults *ConfigWrapper) {
 	if !reflect.DeepEqual(r.Common.ReplicationFactor, defaults.Common.ReplicationFactor) {
@@ -585,12 +598,12 @@ func betterTSDBShipperDefaults(cfg, defaults *ConfigWrapper, period config.Perio
 // (i.e: not applicable for the index queries cache or for the write dedupe cache).
 func applyFIFOCacheConfig(r *ConfigWrapper) {
 	chunkCacheConfig := r.ChunkStoreConfig.ChunkCacheConfig
-	if !cache.IsRedisSet(chunkCacheConfig) && !cache.IsMemcacheSet(chunkCacheConfig) {
+	if !cache.IsCacheConfigured(chunkCacheConfig) {
 		r.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache = true
 	}
 
 	resultsCacheConfig := r.QueryRange.ResultsCacheConfig.CacheConfig
-	if !cache.IsRedisSet(resultsCacheConfig) && !cache.IsMemcacheSet(resultsCacheConfig) {
+	if !cache.IsCacheConfigured(resultsCacheConfig) {
 		r.QueryRange.ResultsCacheConfig.CacheConfig.EnableFifoCache = true
 		// The query results fifocache is still in Cortex so we couldn't change the flag defaults
 		// so instead we will override them here.

pkg/loki/config_wrapper_test.go

@@ -820,6 +820,25 @@ ingester:
 			assert.Equal(t, 12*time.Second, config.Ingester.LifecyclerConfig.FinalSleep)
 		})
 	})
+
+	t.Run("common groupcache setting is applied to chunk, index, and result caches", func(t *testing.T) {
+		// ensure they are all false by default
+		config, _, _ := configWrapperFromYAML(t, minimalConfig, nil)
+		assert.False(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableGroupCache)
+		assert.False(t, config.StorageConfig.IndexQueriesCacheConfig.EnableGroupCache)
+		assert.False(t, config.QueryRange.ResultsCacheConfig.CacheConfig.EnableGroupCache)
+
+		configFileString := `---
+common:
+  groupcache:
+    enabled: true`
+
+		config, _ = testContext(configFileString, nil)
+
+		assert.True(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableGroupCache)
+		assert.True(t, config.StorageConfig.IndexQueriesCacheConfig.EnableGroupCache)
+		assert.True(t, config.QueryRange.ResultsCacheConfig.CacheConfig.EnableGroupCache)
+	})
 }
 
 func TestDefaultFIFOCacheBehavior(t *testing.T) {
@@ -848,6 +867,18 @@ chunk_store_config:
 			assert.False(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache)
 		})
 
+		t.Run("no FIFO cache enabled by default if GroupCache is set", func(t *testing.T) {
+			configFileString := `---
+common:
+  groupcache:
+    enabled: true`
+
+			config, _, _ := configWrapperFromYAML(t, configFileString, nil)
+			assert.False(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache)
+			assert.False(t, config.QueryRange.ResultsCacheConfig.CacheConfig.EnableFifoCache)
+			assert.True(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableGroupCache)
+		})
+
 		t.Run("FIFO cache is enabled by default if no other cache is set", func(t *testing.T) {
 			config, _, _ := configWrapperFromYAML(t, minimalConfig, nil)
 			assert.True(t, config.ChunkStoreConfig.ChunkCacheConfig.EnableFifoCache)

pkg/storage/chunk/cache/cache.go

@@ -25,7 +25,8 @@ type Cache interface {
 
 // Config for building Caches.
 type Config struct {
-	EnableFifoCache bool `yaml:"enable_fifocache"`
+	EnableFifoCache  bool `yaml:"enable_fifocache"`
+	EnableGroupCache bool `yaml:"enable_groupcache"`
 
 	DefaultValidity time.Duration `yaml:"default_validity"`
 
@@ -61,6 +62,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, description string, f
 	f.IntVar(&cfg.AsyncCacheWriteBackBufferSize, prefix+"max-async-cache-write-back-buffer-size", 500, "The maximum number of enqueued asynchronous writeback cache allowed.")
 	f.DurationVar(&cfg.DefaultValidity, prefix+"default-validity", time.Hour, description+"The default validity of entries for caches unless overridden.")
 	f.BoolVar(&cfg.EnableFifoCache, prefix+"cache.enable-fifocache", false, description+"Enable in-memory cache (auto-enabled for the chunks & query results cache if no other cache is configured).")
+	f.BoolVar(&cfg.EnableGroupCache, prefix+"cache.enable-groupcache", false, description+"Enable distributed in-memory cache.")
 
 	cfg.Prefix = prefix
 }
@@ -85,7 +87,12 @@ func IsRedisSet(cfg Config) bool {
 }
 
 func IsGroupCacheSet(cfg Config) bool {
-	return cfg.GroupCache != nil
+	return cfg.EnableGroupCache
+}
+
+// IsCacheConfigured determines if memcached, redis, or groupcache have been configured
+func IsCacheConfigured(cfg Config) bool {
+	return IsMemcacheSet(cfg) || IsRedisSet(cfg) || IsGroupCacheSet(cfg)
 }
 
 // New creates a new Cache using Config.