apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore(blooms): Clean up settings and make reasonable defaults (#12483)

Browse Source 
Signed-off-by: Christian Haudum <christian.haudum@gmail.com>
pull/12455/head
Christian Haudum 2 years ago committed by GitHub
parent f9350d6415
commit a9345d0fb6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 196 additions and 170 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 261
      docs/sources/configure/_index.md
  2. 2
      pkg/bloomcompactor/bloomcompactor.go
  3. 4
      pkg/bloomcompactor/config.go
  4. 2
      pkg/bloomgateway/bloomgateway.go
  5. 15
      pkg/bloomgateway/config.go
  6. 7
      pkg/loki/loki.go
  7. 2
      pkg/storage/factory.go
  8. 2
      pkg/storage/stores/shipper/bloomshipper/config/config.go
  9. 4
      pkg/storage/stores/shipper/bloomshipper/shipper.go
  10. 51
      pkg/validation/limits.go
  11. 16
      tools/doc-generator/parse/root_blocks.go

261
docs/sources/configure/_index.md
Unescape View File

@ -340,13 +340,14 @@ pattern_ingester:
# object store.
[index_gateway: <index_gateway>]
# The bloom_compactor block configures the Loki bloom compactor server,
# responsible for compacting stream indexes into bloom filters and merging them
# as bloom blocks
# Experimental: The bloom_compactor block configures the Loki bloom compactor
# server, responsible for compacting stream indexes into bloom filters and
# merging them as bloom blocks.
[bloom_compactor: <bloom_compactor>]
# The bloom_gateway block configures the Loki bloom gateway server, responsible
# for serving queries for filtering chunks based on filter expressions.
# Experimental: The bloom_gateway block configures the Loki bloom gateway
# server, responsible for serving queries for filtering chunks based on filter
# expressions.
[bloom_gateway: <bloom_gateway>]
# The storage_config block configures one of many possible stores for both the
@ -964,8 +965,8 @@ The `frontend` block configures the Loki query-frontend.
# CLI flag: -frontend.scheduler-worker-concurrency
[scheduler_worker_concurrency: <int> | default = 5]
# The grpc_client block configures the gRPC client used to communicate between
# two Loki components.
# The grpc_client block configures the gRPC client used to communicate between a
# client and server component in Loki.
# The CLI flags prefix for this block configuration is:
# frontend.grpc-client-config
[grpc_client_config: <grpc_client>]
@ -1013,7 +1014,8 @@ The `query_range` block configures the query splitting and caching in the Loki q
[align_queries_with_step: <boolean> | default = false]
results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is: frontend
[cache: <cache_config>]
@ -1048,7 +1050,8 @@ results_cache:
# If a cache config is not specified and cache_index_stats_results is true, the
# config for the results cache is used.
index_stats_results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# frontend.index-stats-results-cache
[cache: <cache_config>]
@ -1065,7 +1068,8 @@ index_stats_results_cache:
# If a cache config is not specified and cache_volume_results is true, the
# config for the results cache is used.
volume_results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# frontend.volume-results-cache
[cache: <cache_config>]
@ -1082,7 +1086,8 @@ volume_results_cache:
# If a cache config is not specified and cache_instant_metric_results is true,
# the config for the results cache is used.
instant_metric_results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# frontend.instant-metric-results-cache
[cache: <cache_config>]
@ -1104,7 +1109,8 @@ instant_metric_results_cache:
# If series_results_cache is not configured and cache_series_results is true,
# the config for the results cache is used.
series_results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# frontend.series-results-cache
[cache: <cache_config>]
@ -1121,7 +1127,8 @@ series_results_cache:
# If label_results_cache is not configured and cache_label_results is true, the
# config for the results cache is used.
label_results_cache:
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# frontend.label-results-cache
[cache: <cache_config>]
@ -1148,8 +1155,8 @@ The `ruler` block configures the Loki ruler.
# Labels to add to all alerts.
[external_labels: <list of Labels>]
# The grpc_client block configures the gRPC client used to communicate between
# two Loki components.
# The grpc_client block configures the gRPC client used to communicate between a
# client and server component in Loki.
# The CLI flags prefix for this block configuration is: ruler.client
[ruler_client: <grpc_client>]
@ -1979,74 +1986,6 @@ ring:
[instance_enable_ipv6: <boolean> | default = false]
```
### bloom_gateway
The `bloom_gateway` block configures the Loki bloom gateway server, responsible for serving queries for filtering chunks based on filter expressions.
```yaml
# Flag to enable or disable the bloom gateway component globally.
# CLI flag: -bloom-gateway.enabled
[enabled: <boolean> | default = false]
client:
# Configures the behavior of the connection pool.
pool_config:
# How frequently to clean up clients for servers that have gone away or are
# unhealthy.
# CLI flag: -bloom-gateway-client.pool.check-interval
[check_interval: <duration> | default = 10s]
# Run a health check on each server during periodic cleanup.
# CLI flag: -bloom-gateway-client.pool.enable-health-check
[enable_health_check: <boolean> | default = true]
# Timeout for the health check if health check is enabled.
# CLI flag: -bloom-gateway-client.pool.health-check-timeout
[health_check_timeout: <duration> | default = 1s]
# The grpc_client block configures the gRPC client used to communicate between
# two Loki components.
# The CLI flags prefix for this block configuration is:
# bloom-gateway-client.grpc
[grpc_client_config: <grpc_client>]
results_cache:
# The cache block configures the cache backend.
# The CLI flags prefix for this block configuration is:
# bloom-gateway-client.cache
[cache: <cache_config>]
# Use compression in cache. The default is an empty value '', which disables
# compression. Supported values are: 'snappy' and ''.
# CLI flag: -bloom-gateway-client.cache.compression
[compression: <string> | default = ""]
# Flag to control whether to cache bloom gateway client requests/responses.
# CLI flag: -bloom-gateway-client.cache_results
[cache_results: <boolean> | default = false]
# Comma separated addresses list in DNS Service Discovery format:
# https://grafana.com/docs/mimir/latest/configure/about-dns-service-discovery/#supported-discovery-modes
# CLI flag: -bloom-gateway-client.addresses
[addresses: <string> | default = ""]
# Number of workers to use for filtering chunks concurrently.
# CLI flag: -bloom-gateway.worker-concurrency
[worker_concurrency: <int> | default = 4]
# Number of blocks processed concurrently on a single worker.
# CLI flag: -bloom-gateway.block-query-concurrency
[block_query_concurrency: <int> | default = 4]
# Maximum number of outstanding tasks per tenant.
# CLI flag: -bloom-gateway.max-outstanding-per-tenant
[max_outstanding_per_tenant: <int> | default = 1024]
# How many tasks are multiplexed at once.
# CLI flag: -bloom-gateway.num-multiplex-tasks
[num_multiplex_tasks: <int> | default = 512]
```
### storage_config
The `storage_config` block configures one of many possible stores for both the index and chunks. Which configuration to be picked should be defined in schema_config block.
@ -2083,7 +2022,7 @@ bigtable:
[instance: <string> | default = ""]
# The grpc_client block configures the gRPC client used to communicate between
# two Loki components.
# a client and server component in Loki.
# The CLI flags prefix for this block configuration is: bigtable
[grpc_client_config: <grpc_client>]
@ -2328,7 +2267,8 @@ congestion_control:
# CLI flag: -store.object-prefix
[object_prefix: <string> | default = ""]
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is: store.index-cache-read
[index_queries_cache_config: <cache_config>]
@ -2373,7 +2313,7 @@ boltdb_shipper:
index_gateway_client:
# The grpc_client block configures the gRPC client used to communicate
# between two Loki components.
# between a client and server component in Loki.
# The CLI flags prefix for this block configuration is:
# boltdb.shipper.index-gateway-client.grpc
[grpc_client_config: <grpc_client>]
@ -2428,7 +2368,7 @@ tsdb_shipper:
index_gateway_client:
# The grpc_client block configures the gRPC client used to communicate
# between two Loki components.
# between a client and server component in Loki.
# The CLI flags prefix for this block configuration is:
# tsdb.shipper.index-gateway-client.grpc
[grpc_client_config: <grpc_client>]
@ -2451,8 +2391,8 @@ tsdb_shipper:
[ingesterdbretainperiod: <duration>]
# Configures the bloom shipper component, which contains the store abstraction
# to fetch bloom filters from and put them to object storage.
# Experimental: Configures the bloom shipper component, which contains the store
# abstraction to fetch bloom filters from and put them to object storage.
bloom_shipper:
# Working directory to store downloaded bloom blocks. Supports multiple
# directories, separated by comma.
@ -2464,9 +2404,10 @@ bloom_shipper:
# CLI flag: -bloom.max-query-page-size
[max_query_page_size: <int> | default = 64MiB]
# The amount of maximum concurrent bloom blocks downloads.
# The amount of maximum concurrent bloom blocks downloads. Usually set to 2x
# number of CPU cores.
# CLI flag: -bloom.download-parallelism
[download_parallelism: <int> | default = 16]
[download_parallelism: <int> | default = 8]
blocks_cache:
# Cache for bloom blocks. Soft limit of the cache in bytes. Exceeding this
@ -2485,7 +2426,8 @@ bloom_shipper:
# CLI flag: -bloom.blocks-cache.ttl
[ttl: <duration> | default = 24h]
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is: bloom.metas-cache
[metas_cache: <cache_config>]
```
@ -2495,11 +2437,13 @@ bloom_shipper:
The `chunk_store_config` block configures how chunks will be cached and how long to wait before saving them to the backing store.
```yaml
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is: store.chunks-cache
[chunk_cache_config: <cache_config>]
# The cache block configures the cache backend.
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is: store.chunks-cache-l2
[chunk_cache_config_l2: <cache_config>]
@ -2699,7 +2643,7 @@ compactor_ring:
### bloom_compactor
The `bloom_compactor` block configures the Loki bloom compactor server, responsible for compacting stream indexes into bloom filters and merging them as bloom blocks
Experimental: The `bloom_compactor` block configures the Loki bloom compactor server, responsible for compacting stream indexes into bloom filters and merging them as bloom blocks.
```yaml
# Defines the ring to be used by the bloom-compactor servers. In case this isn't
@ -2845,6 +2789,77 @@ retention:
[max_lookback_days: <int> | default = 365]
```
### bloom_gateway
Experimental: The `bloom_gateway` block configures the Loki bloom gateway server, responsible for serving queries for filtering chunks based on filter expressions.
```yaml
# Flag to enable or disable the bloom gateway component globally.
# CLI flag: -bloom-gateway.enabled
[enabled: <boolean> | default = false]
client:
# Configures the behavior of the connection pool.
pool_config:
# How frequently to clean up clients for servers that have gone away or are
# unhealthy.
# CLI flag: -bloom-gateway-client.pool.check-interval
[check_interval: <duration> | default = 10s]
# Run a health check on each server during periodic cleanup.
# CLI flag: -bloom-gateway-client.pool.enable-health-check
[enable_health_check: <boolean> | default = true]
# Timeout for the health check if health check is enabled.
# CLI flag: -bloom-gateway-client.pool.health-check-timeout
[health_check_timeout: <duration> | default = 1s]
# The grpc_client block configures the gRPC client used to communicate between
# a client and server component in Loki.
# The CLI flags prefix for this block configuration is:
# bloom-gateway-client.grpc
[grpc_client_config: <grpc_client>]
results_cache:
# The cache_config block configures the cache backend for a specific Loki
# component.
# The CLI flags prefix for this block configuration is:
# bloom-gateway-client.cache
[cache: <cache_config>]
# Use compression in cache. The default is an empty value '', which disables
# compression. Supported values are: 'snappy' and ''.
# CLI flag: -bloom-gateway-client.cache.compression
[compression: <string> | default = ""]
# Flag to control whether to cache bloom gateway client requests/responses.
# CLI flag: -bloom-gateway-client.cache_results
[cache_results: <boolean> | default = false]
# Comma separated addresses list in DNS Service Discovery format:
# https://grafana.com/docs/mimir/latest/configure/about-dns-service-discovery/#supported-discovery-modes
# CLI flag: -bloom-gateway-client.addresses
[addresses: <string> | default = ""]
# Number of workers to use for filtering chunks concurrently. Usually set to 1x
# number of CPU cores.
# CLI flag: -bloom-gateway.worker-concurrency
[worker_concurrency: <int> | default = 4]
# Number of blocks processed concurrently on a single worker. Usually set to 2x
# number of CPU cores.
# CLI flag: -bloom-gateway.block-query-concurrency
[block_query_concurrency: <int> | default = 8]
# Maximum number of outstanding tasks per tenant.
# CLI flag: -bloom-gateway.max-outstanding-per-tenant
[max_outstanding_per_tenant: <int> | default = 1024]
# How many tasks are multiplexed at once.
# CLI flag: -bloom-gateway.num-multiplex-tasks
[num_multiplex_tasks: <int> | default = 512]
```
### limits_config
The `limits_config` block configures global and per-tenant limits in Loki. The values here can be overridden in the `overrides` section of the runtime_config file
@ -3308,54 +3323,54 @@ shard_streams:
# CLI flag: -index-gateway.shard-size
[index_gateway_shard_size: <int> | default = 0]
# The shard size defines how many bloom gateways should be used by a tenant for
# querying.
# Experimental. The shard size defines how many bloom gateways should be used by
# a tenant for querying.
# CLI flag: -bloom-gateway.shard-size
[bloom_gateway_shard_size: <int> | default = 0]
# Whether to use the bloom gateway component in the read path to filter chunks.
# Experimental. Whether to use the bloom gateway component in the read path to
# filter chunks.
# CLI flag: -bloom-gateway.enable-filtering
[bloom_gateway_enable_filtering: <boolean> | default = false]
# The shard size defines how many bloom compactors should be used by a tenant
# when computing blooms. If it's set to 0, shuffle sharding is disabled.
# Experimental. Interval for computing the cache key in the Bloom Gateway.
# CLI flag: -bloom-gateway.cache-key-interval
[bloom_gateway_cache_key_interval: <duration> | default = 15m]
# Experimental. The shard size defines how many bloom compactors should be used
# by a tenant when computing blooms. If it's set to 0, shuffle sharding is
# disabled.
# CLI flag: -bloom-compactor.shard-size
[bloom_compactor_shard_size: <int> | default = 0]
# Whether to compact chunks into bloom filters.
# Experimental. Whether to compact chunks into bloom filters.
# CLI flag: -bloom-compactor.enable-compaction
[bloom_compactor_enable_compaction: <boolean> | default = false]
# Length of the n-grams created when computing blooms from log lines.
# Experimental. The maximum bloom block size. A value of 0 sets an unlimited
# size. Default is 200MB. The actual block size might exceed this limit since
# blooms will be added to blocks until the block exceeds the maximum block size.
# CLI flag: -bloom-compactor.max-block-size
[bloom_compactor_max_block_size: <int> | default = 200MB]
# Experimental. Length of the n-grams created when computing blooms from log
# lines.
# CLI flag: -bloom-compactor.ngram-length
[bloom_ngram_length: <int> | default = 4]
# Skip factor for the n-grams created when computing blooms from log lines.
# Experimental. Skip factor for the n-grams created when computing blooms from
# log lines.
# CLI flag: -bloom-compactor.ngram-skip
[bloom_ngram_skip: <int> | default = 1]
# Scalable Bloom Filter desired false-positive rate.
# Experimental. Scalable Bloom Filter desired false-positive rate.
# CLI flag: -bloom-compactor.false-positive-rate
[bloom_false_positive_rate: <float> | default = 0.01]
# Compression algorithm for bloom block pages.
# Experimental. Compression algorithm for bloom block pages.
# CLI flag: -bloom-compactor.block-encoding
[bloom_block_encoding: <string> | default = "none"]
# Maximum number of blocks will be downloaded in parallel by the Bloom Gateway.
# CLI flag: -bloom-gateway.blocks-downloading-parallelism
[bloom_gateway_blocks_downloading_parallelism: <int> | default = 50]
# Interval for computing the cache key in the Bloom Gateway.
# CLI flag: -bloom-gateway.cache-key-interval
[bloom_gateway_cache_key_interval: <duration> | default = 15m]
# The maximum bloom block size. A value of 0 sets an unlimited size. Default is
# 200MB. The actual block size might exceed this limit since blooms will be
# added to blocks until the block exceeds the maximum block size.
# CLI flag: -bloom-compactor.max-block-size
[bloom_compactor_max_block_size: <int> | default = 200MB]
# Allow user to send structured metadata in push payload.
# CLI flag: -validation.allow-structured-metadata
[allow_structured_metadata: <boolean> | default = true]
@ -3419,8 +3434,8 @@ The `frontend_worker` configures the worker - running within the Loki querier -
# CLI flag: -querier.id
[id: <string> | default = ""]
# The grpc_client block configures the gRPC client used to communicate between
# two Loki components.
# The grpc_client block configures the gRPC client used to communicate between a
# client and server component in Loki.
# The CLI flags prefix for this block configuration is: querier.frontend-client
[grpc_client_config: <grpc_client>]
```
@ -4396,7 +4411,7 @@ When a memberlist config with atleast 1 join_members is defined, kvstore of type
### grpc_client
The `grpc_client` block configures the gRPC client used to communicate between two Loki components. The supported CLI flags `<prefix>` used to reference this configuration block are:
The `grpc_client` block configures the gRPC client used to communicate between a client and server component in Loki. The supported CLI flags `<prefix>` used to reference this configuration block are:
- `bigtable`
- `bloom-gateway-client.grpc`
@ -4614,7 +4629,7 @@ The TLS configuration.
### cache_config
The cache block configures the cache backend. The supported CLI flags `<prefix>` used to reference this configuration block are:
The `cache_config` block configures the cache backend for a specific Loki component. The supported CLI flags `<prefix>` used to reference this configuration block are:
- `bloom-gateway-client.cache`
- `bloom.metas-cache`

2
pkg/bloomcompactor/bloomcompactor.go
Unescape View File

@ -22,6 +22,7 @@ import (
"github.com/grafana/loki/v3/pkg/storage/config"
"github.com/grafana/loki/v3/pkg/storage/stores"
"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
utillog "github.com/grafana/loki/v3/pkg/util/log"
util_ring "github.com/grafana/loki/v3/pkg/util/ring"
)
@ -72,6 +73,7 @@ func New(
logger log.Logger,
r prometheus.Registerer,
) (*Compactor, error) {
utillog.WarnExperimentalUse("Bloom Compactor", logger)
c := &Compactor{
cfg: cfg,
schemaCfg: schemaCfg,

4
pkg/bloomcompactor/config.go
Unescape View File

@ -68,6 +68,10 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
}
func (cfg *Config) Validate() error {
if !cfg.Enabled {
return nil
}
if err := cfg.RetentionConfig.Validate(); err != nil {
return err
}

2
pkg/bloomgateway/bloomgateway.go
Unescape View File

@ -62,6 +62,7 @@ import (
"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
"github.com/grafana/loki/v3/pkg/util"
"github.com/grafana/loki/v3/pkg/util/constants"
utillog "github.com/grafana/loki/v3/pkg/util/log"
)
var errGatewayUnhealthy = errors.New("bloom-gateway is unhealthy in the ring")
@ -108,6 +109,7 @@ func (l *fixedQueueLimits) MaxConsumers(_ string, _ int) int {
// New returns a new instance of the Bloom Gateway.
func New(cfg Config, store bloomshipper.Store, logger log.Logger, reg prometheus.Registerer) (*Gateway, error) {
utillog.WarnExperimentalUse("Bloom Gateway", logger)
g := &Gateway{
cfg: cfg,
logger: logger,

15
pkg/bloomgateway/config.go
Unescape View File

@ -25,8 +25,8 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
// RegisterFlagsWithPrefix registers flags for the Bloom Gateway configuration with a common prefix.
func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
f.BoolVar(&cfg.Enabled, prefix+"enabled", false, "Flag to enable or disable the bloom gateway component globally.")
f.IntVar(&cfg.WorkerConcurrency, prefix+"worker-concurrency", 4, "Number of workers to use for filtering chunks concurrently.")
f.IntVar(&cfg.BlockQueryConcurrency, prefix+"block-query-concurrency", 4, "Number of blocks processed concurrently on a single worker.")
f.IntVar(&cfg.WorkerConcurrency, prefix+"worker-concurrency", 4, "Number of workers to use for filtering chunks concurrently. Usually set to 1x number of CPU cores.")
f.IntVar(&cfg.BlockQueryConcurrency, prefix+"block-query-concurrency", 8, "Number of blocks processed concurrently on a single worker. Usually set to 2x number of CPU cores.")
f.IntVar(&cfg.MaxOutstandingPerTenant, prefix+"max-outstanding-per-tenant", 1024, "Maximum number of outstanding tasks per tenant.")
f.IntVar(&cfg.NumMultiplexItems, prefix+"num-multiplex-tasks", 512, "How many tasks are multiplexed at once.")
// TODO(chaudum): Figure out what the better place is for registering flags
@ -34,9 +34,18 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
cfg.Client.RegisterFlags(f)
}
func (cfg *Config) Validate() error {
if !cfg.Enabled {
return nil
}
if err := cfg.Client.Validate(); err != nil {
return err
}
return nil
}
type Limits interface {
CacheLimits
BloomGatewayShardSize(tenantID string) int
BloomGatewayEnabled(tenantID string) bool
BloomGatewayBlocksDownloadingParallelism(tenantID string) int
}

7
pkg/loki/loki.go
Unescape View File

@ -90,8 +90,8 @@ type Config struct {
Ingester ingester.Config `yaml:"ingester,omitempty"`
Pattern pattern.Config `yaml:"pattern_ingester,omitempty"`
IndexGateway indexgateway.Config `yaml:"index_gateway"`
BloomCompactor bloomcompactor.Config `yaml:"bloom_compactor"`
BloomGateway bloomgateway.Config `yaml:"bloom_gateway"`
BloomCompactor bloomcompactor.Config `yaml:"bloom_compactor,omitempty" category:"experimental"`
BloomGateway bloomgateway.Config `yaml:"bloom_gateway,omitempty" category:"experimental"`
StorageConfig storage.Config `yaml:"storage_config,omitempty"`
ChunkStoreConfig config.ChunkStoreConfig `yaml:"chunk_store_config,omitempty"`
SchemaConfig config.SchemaConfig `yaml:"schema_config,omitempty"`
@ -269,6 +269,9 @@ func (c *Config) Validate() error {
if err := c.BloomCompactor.Validate(); err != nil {
return errors.Wrap(err, "invalid bloom_compactor config")
}
if err := c.BloomGateway.Validate(); err != nil {
return errors.Wrap(err, "invalid bloom_gateway config")
}
if err := c.Pattern.Validate(); err != nil {
return errors.Wrap(err, "invalid pattern_ingester config")

2
pkg/storage/factory.go
Unescape View File

@ -336,7 +336,7 @@ type Config struct {
MaxChunkBatchSize int `yaml:"max_chunk_batch_size"`
BoltDBShipperConfig boltdb.IndexCfg `yaml:"boltdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in the form of boltdb files. Required fields only required when boltdb-shipper is defined in config."`
TSDBShipperConfig indexshipper.Config `yaml:"tsdb_shipper" doc:"description=Configures storing index in an Object Store (GCS/S3/Azure/Swift/COS/Filesystem) in a prometheus TSDB-like format. Required fields only required when TSDB is defined in config."`
BloomShipperConfig bloomshipperconfig.Config `yaml:"bloom_shipper" doc:"description=Configures the bloom shipper component, which contains the store abstraction to fetch bloom filters from and put them to object storage."`
BloomShipperConfig bloomshipperconfig.Config `yaml:"bloom_shipper" category:"experimental" doc:"description=Experimental: Configures the bloom shipper component, which contains the store abstraction to fetch bloom filters from and put them to object storage."`
// Config for using AsyncStore when using async index stores like `boltdb-shipper`.
// It is required for getting chunk ids of recently flushed chunks from the ingesters.

2
pkg/storage/stores/shipper/bloomshipper/config/config.go
Unescape View File

@ -28,7 +28,7 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
f.Var(&c.WorkingDirectory, prefix+"shipper.working-directory", "Working directory to store downloaded bloom blocks. Supports multiple directories, separated by comma.")
_ = c.MaxQueryPageSize.Set("64MiB") // default should match the one set in pkg/storage/bloom/v1/bloom.go
f.Var(&c.MaxQueryPageSize, prefix+"max-query-page-size", "Maximum size of bloom pages that should be queried. Larger pages than this limit are skipped when querying blooms to limit memory usage.")
f.IntVar(&c.DownloadParallelism, prefix+"download-parallelism", 16, "The amount of maximum concurrent bloom blocks downloads.")
f.IntVar(&c.DownloadParallelism, prefix+"download-parallelism", 8, "The amount of maximum concurrent bloom blocks downloads. Usually set to 2x number of CPU cores.")
c.BlocksCache.RegisterFlagsWithPrefixAndDefaults(prefix+"blocks-cache.", "Cache for bloom blocks. ", f, 24*time.Hour)
c.MetasCache.RegisterFlagsWithPrefix(prefix+"metas-cache.", "Cache for bloom metas. ", f)

4
pkg/storage/stores/shipper/bloomshipper/shipper.go
Unescape View File

@ -19,10 +19,6 @@ type Shipper struct {
store Store
}
type Limits interface {
BloomGatewayBlocksDownloadingParallelism(tenantID string) int
}
func NewShipper(client Store) *Shipper {
return &Shipper{store: client}
}

51
pkg/validation/limits.go
Unescape View File

@ -194,18 +194,18 @@ type Limits struct {
IndexGatewayShardSize int `yaml:"index_gateway_shard_size" json:"index_gateway_shard_size"`
BloomGatewayShardSize int `yaml:"bloom_gateway_shard_size" json:"bloom_gateway_shard_size"`
BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering"`
BloomCompactorShardSize int `yaml:"bloom_compactor_shard_size" json:"bloom_compactor_shard_size"`
BloomCompactorEnabled bool `yaml:"bloom_compactor_enable_compaction" json:"bloom_compactor_enable_compaction"`
BloomNGramLength int `yaml:"bloom_ngram_length" json:"bloom_ngram_length"`
BloomNGramSkip int `yaml:"bloom_ngram_skip" json:"bloom_ngram_skip"`
BloomFalsePositiveRate float64 `yaml:"bloom_false_positive_rate" json:"bloom_false_positive_rate"`
BloomBlockEncoding string `yaml:"bloom_block_encoding" json:"bloom_block_encoding"`
BloomGatewayBlocksDownloadingParallelism int `yaml:"bloom_gateway_blocks_downloading_parallelism" json:"bloom_gateway_blocks_downloading_parallelism"`
BloomGatewayCacheKeyInterval time.Duration `yaml:"bloom_gateway_cache_key_interval" json:"bloom_gateway_cache_key_interval"`
BloomCompactorMaxBlockSize flagext.ByteSize `yaml:"bloom_compactor_max_block_size" json:"bloom_compactor_max_block_size"`
BloomGatewayShardSize int `yaml:"bloom_gateway_shard_size" json:"bloom_gateway_shard_size" category:"experimental"`
BloomGatewayEnabled bool `yaml:"bloom_gateway_enable_filtering" json:"bloom_gateway_enable_filtering" category:"experimental"`
BloomGatewayCacheKeyInterval time.Duration `yaml:"bloom_gateway_cache_key_interval" json:"bloom_gateway_cache_key_interval" category:"experimental"`
BloomCompactorShardSize int `yaml:"bloom_compactor_shard_size" json:"bloom_compactor_shard_size" category:"experimental"`
BloomCompactorEnabled bool `yaml:"bloom_compactor_enable_compaction" json:"bloom_compactor_enable_compaction" category:"experimental"`
BloomCompactorMaxBlockSize flagext.ByteSize `yaml:"bloom_compactor_max_block_size" json:"bloom_compactor_max_block_size" category:"experimental"`
BloomNGramLength int `yaml:"bloom_ngram_length" json:"bloom_ngram_length" category:"experimental"`
BloomNGramSkip int `yaml:"bloom_ngram_skip" json:"bloom_ngram_skip" category:"experimental"`
BloomFalsePositiveRate float64 `yaml:"bloom_false_positive_rate" json:"bloom_false_positive_rate" category:"experimental"`
BloomBlockEncoding string `yaml:"bloom_block_encoding" json:"bloom_block_encoding" category:"experimental"`
AllowStructuredMetadata bool `yaml:"allow_structured_metadata,omitempty" json:"allow_structured_metadata,omitempty" doc:"description=Allow user to send structured metadata in push payload."`
MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."`
@ -358,21 +358,20 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
f.IntVar(&l.IndexGatewayShardSize, "index-gateway.shard-size", 0, "The shard size defines how many index gateways should be used by a tenant for querying. If the global shard factor is 0, the global shard factor is set to the deprecated -replication-factor for backwards compatibility reasons.")
f.IntVar(&l.BloomGatewayShardSize, "bloom-gateway.shard-size", 0, "The shard size defines how many bloom gateways should be used by a tenant for querying.")
f.BoolVar(&l.BloomGatewayEnabled, "bloom-gateway.enable-filtering", false, "Whether to use the bloom gateway component in the read path to filter chunks.")
f.IntVar(&l.BloomCompactorShardSize, "bloom-compactor.shard-size", 0, "The shard size defines how many bloom compactors should be used by a tenant when computing blooms. If it's set to 0, shuffle sharding is disabled.")
f.BoolVar(&l.BloomCompactorEnabled, "bloom-compactor.enable-compaction", false, "Whether to compact chunks into bloom filters.")
f.IntVar(&l.BloomNGramLength, "bloom-compactor.ngram-length", 4, "Length of the n-grams created when computing blooms from log lines.")
f.IntVar(&l.BloomNGramSkip, "bloom-compactor.ngram-skip", 1, "Skip factor for the n-grams created when computing blooms from log lines.")
f.Float64Var(&l.BloomFalsePositiveRate, "bloom-compactor.false-positive-rate", 0.01, "Scalable Bloom Filter desired false-positive rate.")
f.StringVar(&l.BloomBlockEncoding, "bloom-compactor.block-encoding", "none", "Compression algorithm for bloom block pages.")
f.IntVar(&l.BloomGatewayBlocksDownloadingParallelism, "bloom-gateway.blocks-downloading-parallelism", 50, "Maximum number of blocks will be downloaded in parallel by the Bloom Gateway.")
f.DurationVar(&l.BloomGatewayCacheKeyInterval, "bloom-gateway.cache-key-interval", 15*time.Minute, "Interval for computing the cache key in the Bloom Gateway.")
f.IntVar(&l.BloomGatewayShardSize, "bloom-gateway.shard-size", 0, "Experimental. The shard size defines how many bloom gateways should be used by a tenant for querying.")
f.BoolVar(&l.BloomGatewayEnabled, "bloom-gateway.enable-filtering", false, "Experimental. Whether to use the bloom gateway component in the read path to filter chunks.")
f.IntVar(&l.BloomCompactorShardSize, "bloom-compactor.shard-size", 0, "Experimental. The shard size defines how many bloom compactors should be used by a tenant when computing blooms. If it's set to 0, shuffle sharding is disabled.")
f.BoolVar(&l.BloomCompactorEnabled, "bloom-compactor.enable-compaction", false, "Experimental. Whether to compact chunks into bloom filters.")
f.IntVar(&l.BloomNGramLength, "bloom-compactor.ngram-length", 4, "Experimental. Length of the n-grams created when computing blooms from log lines.")
f.IntVar(&l.BloomNGramSkip, "bloom-compactor.ngram-skip", 1, "Experimental. Skip factor for the n-grams created when computing blooms from log lines.")
f.Float64Var(&l.BloomFalsePositiveRate, "bloom-compactor.false-positive-rate", 0.01, "Experimental. Scalable Bloom Filter desired false-positive rate.")
f.StringVar(&l.BloomBlockEncoding, "bloom-compactor.block-encoding", "none", "Experimental. Compression algorithm for bloom block pages.")
f.DurationVar(&l.BloomGatewayCacheKeyInterval, "bloom-gateway.cache-key-interval", 15*time.Minute, "Experimental. Interval for computing the cache key in the Bloom Gateway.")
_ = l.BloomCompactorMaxBlockSize.Set(defaultBloomCompactorMaxBlockSize)
f.Var(&l.BloomCompactorMaxBlockSize, "bloom-compactor.max-block-size",
fmt.Sprintf(
"The maximum bloom block size. A value of 0 sets an unlimited size. Default is %s. The actual block size might exceed this limit since blooms will be added to blocks until the block exceeds the maximum block size.",
"Experimental. The maximum bloom block size. A value of 0 sets an unlimited size. Default is %s. The actual block size might exceed this limit since blooms will be added to blocks until the block exceeds the maximum block size.",
defaultBloomCompactorMaxBlockSize,
),
)
@ -938,10 +937,6 @@ func (o *Overrides) BloomGatewayShardSize(userID string) int {
return o.getOverridesForUser(userID).BloomGatewayShardSize
}
func (o *Overrides) BloomGatewayBlocksDownloadingParallelism(userID string) int {
return o.getOverridesForUser(userID).BloomGatewayBlocksDownloadingParallelism
}
func (o *Overrides) BloomGatewayCacheKeyInterval(userID string) time.Duration {
return o.getOverridesForUser(userID).BloomGatewayCacheKeyInterval
}

16
tools/doc-generator/parse/root_blocks.go
Unescape View File

@ -101,11 +101,6 @@ var (
StructType: []reflect.Type{reflect.TypeOf(indexgateway.Config{})},
Desc: "The index_gateway block configures the Loki index gateway server, responsible for serving index queries without the need to constantly interact with the object store.",
},
{
Name: "bloom_gateway",
StructType: []reflect.Type{reflect.TypeOf(bloomgateway.Config{})},
Desc: "The bloom_gateway block configures the Loki bloom gateway server, responsible for serving queries for filtering chunks based on filter expressions.",
},
{
Name: "storage_config",
StructType: []reflect.Type{reflect.TypeOf(storage.Config{})},
@ -129,7 +124,12 @@ var (
{
Name: "bloom_compactor",
StructType: []reflect.Type{reflect.TypeOf(bloomcompactor.Config{})},
Desc: "The bloom_compactor block configures the Loki bloom compactor server, responsible for compacting stream indexes into bloom filters and merging them as bloom blocks",
Desc: "Experimental: The bloom_compactor block configures the Loki bloom compactor server, responsible for compacting stream indexes into bloom filters and merging them as bloom blocks.",
},
{
Name: "bloom_gateway",
StructType: []reflect.Type{reflect.TypeOf(bloomgateway.Config{})},
Desc: "Experimental: The bloom_gateway block configures the Loki bloom gateway server, responsible for serving queries for filtering chunks based on filter expressions.",
},
{
Name: "limits_config",
@ -197,7 +197,7 @@ When a memberlist config with atleast 1 join_members is defined, kvstore of type
{
Name: "grpc_client",
StructType: []reflect.Type{reflect.TypeOf(grpcclient.Config{})},
Desc: "The grpc_client block configures the gRPC client used to communicate between two Loki components.",
Desc: "The grpc_client block configures the gRPC client used to communicate between a client and server component in Loki.",
},
// TLS config
{
@ -209,7 +209,7 @@ When a memberlist config with atleast 1 join_members is defined, kvstore of type
{
Name: "cache_config",
StructType: []reflect.Type{reflect.TypeOf(cache.Config{})},
Desc: "The cache block configures the cache backend.",
Desc: "The cache_config block configures the cache backend for a specific Loki component.",
},
// Schema periodic config
{

Write Preview
Loading…
Cancel
Save