feat(blooms): Add in-memory LRU cache for meta files (#12862)

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

docs/sources/shared/configuration.md

@@ -5280,6 +5280,26 @@ bloom_shipper:
   # component.
   # The CLI flags prefix for this block configuration is: bloom.metas-cache
   [metas_cache: <cache_config>]
+
+  metas_lru_cache:
+    # In-memory LRU cache for bloom metas. Whether embedded cache is enabled.
+    # CLI flag: -bloom.metas-lru-cache.enabled
+    [enabled: <boolean> | default = false]
+
+    # In-memory LRU cache for bloom metas. Maximum memory size of the cache in
+    # MB.
+    # CLI flag: -bloom.metas-lru-cache.max-size-mb
+    [max_size_mb: <int> | default = 100]
+
+    # In-memory LRU cache for bloom metas. Maximum number of entries in the
+    # cache.
+    # CLI flag: -bloom.metas-lru-cache.max-size-items
+    [max_size_items: <int> | default = 0]
+
+    # In-memory LRU cache for bloom metas. The time to live for items in the
+    # cache before they get purged.
+    # CLI flag: -bloom.metas-lru-cache.ttl
+    [ttl: <duration> | default = 1h]
 ```
 
 ### swift_storage_config

pkg/loki/modules.go

@@ -729,6 +729,14 @@ func (t *Loki) initBloomStore() (services.Service, error) {
 	var metasCache cache.Cache
 	if cache.IsCacheConfigured(bsCfg.MetasCache) {
 		metasCache, err = cache.New(bsCfg.MetasCache, reg, logger, stats.BloomMetasCache, constants.Loki)
+
+		// always enable LRU cache
+		lruCfg := bsCfg.MetasLRUCache
+		lruCfg.Enabled = true
+		lruCfg.PurgeInterval = 1 * time.Minute
+		lruCache := cache.NewEmbeddedCache("inmemory-metas-lru", lruCfg, reg, logger, stats.BloomMetasCache)
+
+		metasCache = cache.NewTiered([]cache.Cache{lruCache, metasCache})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create metas cache: %w", err)
 		}

pkg/storage/stores/shipper/bloomshipper/config/config.go

@@ -12,11 +12,12 @@ import (
 )
 
 type Config struct {
-	WorkingDirectory    flagext.StringSliceCSV `yaml:"working_directory"`
-	MaxQueryPageSize    flagext.Bytes          `yaml:"max_query_page_size"`
-	DownloadParallelism int                    `yaml:"download_parallelism"`
-	BlocksCache         BlocksCacheConfig      `yaml:"blocks_cache"`
-	MetasCache          cache.Config           `yaml:"metas_cache"`
+	WorkingDirectory    flagext.StringSliceCSV    `yaml:"working_directory"`
+	MaxQueryPageSize    flagext.Bytes             `yaml:"max_query_page_size"`
+	DownloadParallelism int                       `yaml:"download_parallelism"`
+	BlocksCache         BlocksCacheConfig         `yaml:"blocks_cache"`
+	MetasCache          cache.Config              `yaml:"metas_cache"`
+	MetasLRUCache       cache.EmbeddedCacheConfig `yaml:"metas_lru_cache"`
 
 	// This will always be set to true when flags are registered.
 	// In tests, where config is created as literal, it can be set manually.
@@ -32,6 +33,7 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&c.DownloadParallelism, prefix+"download-parallelism", 8, "The amount of maximum concurrent bloom blocks downloads. Usually set to 2x number of CPU cores.")
 	c.BlocksCache.RegisterFlagsWithPrefixAndDefaults(prefix+"blocks-cache.", "Cache for bloom blocks. ", f, 24*time.Hour)
 	c.MetasCache.RegisterFlagsWithPrefix(prefix+"metas-cache.", "Cache for bloom metas. ", f)
+	c.MetasLRUCache.RegisterFlagsWithPrefix(prefix+"metas-lru-cache.", "In-memory LRU cache for bloom metas. ", f)
 
 	// always cache LIST operations
 	c.CacheListOps = true