config: ensure storage config defaults apply to named stores (#9650)

**What this PR does / why we need it**:
Since named store config does not register any flags, storage configs
defined under it do not get the defaults.
For example
[aws_storage_config](https://grafana.com/docs/loki/latest/configuration/#aws_storage_config)
sets the default `storage_class` to `STANDARD`, but the same doesn't get
applied by default when using named stores.

This PR ensures that named storage configs are always assigned default
values when they are unmarshalled by implementing `yaml.Unmarshaler`
interface
pull/8886/head
Ashwanth 2 years ago committed by GitHub
parent 4cebc2d85c
commit 98d1307c7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 137
      pkg/loki/config_wrapper_test.go
  3. 140
      pkg/storage/factory.go
  4. 14
      pkg/storage/factory_test.go
  5. 2
      pkg/storage/store_test.go
  6. 14
      tools/doc-generator/parse/parser.go
  7. 81
      tools/doc-generator/parse/root_blocks.go

@ -45,6 +45,7 @@
* [9463](https://github.com/grafana/loki/pull/9463) **Totalus**: Fix OpenStack Swift client object listing to fetch all the objects properly.
* [9471](https://github.com/grafana/loki/pull/9471) **sandeepsukhani**: query-scheduler: fix query distribution in SSD mode.
* [9495](https://github.com/grafana/loki/pull/9495) **thampiotr**: Promtail: Fix potential goroutine leak in file tailer.
* [9650](https://github.com/grafana/loki/pull/9650) **ashwanthgoli**: Config: ensure storage config defaults apply to named stores.
* [9629](https://github.com/grafana/loki/pull/9629) **periklis**: Fix duplicate label values from ingester streams.
##### Changes

@ -16,10 +16,14 @@ import (
"github.com/grafana/loki/pkg/distributor"
"github.com/grafana/loki/pkg/loki/common"
"github.com/grafana/loki/pkg/storage/bucket/swift"
"github.com/grafana/loki/pkg/storage/chunk/client/alibaba"
"github.com/grafana/loki/pkg/storage/chunk/client/aws"
"github.com/grafana/loki/pkg/storage/chunk/client/azure"
"github.com/grafana/loki/pkg/storage/chunk/client/baidubce"
"github.com/grafana/loki/pkg/storage/chunk/client/gcp"
"github.com/grafana/loki/pkg/storage/chunk/client/ibmcloud"
"github.com/grafana/loki/pkg/storage/chunk/client/local"
"github.com/grafana/loki/pkg/storage/chunk/client/openstack"
"github.com/grafana/loki/pkg/storage/config"
"github.com/grafana/loki/pkg/util"
"github.com/grafana/loki/pkg/util/cfg"
@ -1702,3 +1706,136 @@ common:
assert.Equal(t, []string{"ringsshouldusethis"}, config.CompactorConfig.CompactorRing.InstanceInterfaceNames)
})
}
func TestNamedStores_applyDefaults(t *testing.T) {
namedStoresConfig := `storage_config:
named_stores:
aws:
store-1:
s3: "s3.test"
storage_class: GLACIER
dynamodb:
dynamodb_url: "dynamo.test"
azure:
store-2:
environment: AzureGermanCloud
account_name: foo
container_name: bar
bos:
store-3:
bucket_name: foobar
gcs:
store-4:
bucket_name: foobar
enable_http2: false
cos:
store-5:
endpoint: cos.test
http_config:
idle_conn_timeout: 30s
filesystem:
store-6:
directory: foobar
swift:
store-7:
container_name: foobar
request_timeout: 30s
alibabacloud:
store-8:
bucket: foobar
endpoint: oss.test
`
// make goconst happy
bucketName := "foobar"
config, defaults, err := configWrapperFromYAML(t, namedStoresConfig, nil)
require.NoError(t, err)
nsCfg := config.StorageConfig.NamedStores
t.Run("aws", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.AWS, 1)
// expect the defaults to be set on named store config
expected := defaults.StorageConfig.AWSStorageConfig
assert.NoError(t, expected.DynamoDB.Set("dynamo.test"))
assert.NoError(t, expected.S3.Set("s3.test"))
// override defaults
expected.StorageClass = "GLACIER"
assert.Equal(t, expected, (aws.StorageConfig)(nsCfg.AWS["store-1"]))
})
t.Run("azure", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.Azure, 1)
expected := defaults.StorageConfig.AzureStorageConfig
expected.StorageAccountName = "foo"
expected.ContainerName = "bar"
// override defaults
expected.Environment = "AzureGermanCloud"
assert.Equal(t, expected, (azure.BlobStorageConfig)(nsCfg.Azure["store-2"]))
})
t.Run("bos", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.BOS, 1)
expected := defaults.StorageConfig.BOSStorageConfig
expected.BucketName = bucketName
assert.Equal(t, expected, (baidubce.BOSStorageConfig)(nsCfg.BOS["store-3"]))
})
t.Run("gcs", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.GCS, 1)
expected := defaults.StorageConfig.GCSConfig
expected.BucketName = bucketName
// override defaults
expected.EnableHTTP2 = false
assert.Equal(t, expected, (gcp.GCSConfig)(nsCfg.GCS["store-4"]))
})
t.Run("cos", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.COS, 1)
expected := defaults.StorageConfig.COSConfig
expected.Endpoint = "cos.test"
// override defaults
expected.HTTPConfig.IdleConnTimeout = 30 * time.Second
assert.Equal(t, expected, (ibmcloud.COSConfig)(nsCfg.COS["store-5"]))
})
t.Run("filesystem", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.Filesystem, 1)
expected := defaults.StorageConfig.FSConfig
expected.Directory = bucketName
assert.Equal(t, expected, (local.FSConfig)(nsCfg.Filesystem["store-6"]))
})
t.Run("swift", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.Swift, 1)
expected := defaults.StorageConfig.Swift
expected.ContainerName = bucketName
// override defaults
expected.RequestTimeout = 30 * time.Second
assert.Equal(t, expected, (openstack.SwiftConfig)(nsCfg.Swift["store-7"]))
})
t.Run("alibabacloud", func(t *testing.T) {
assert.Len(t, config.StorageConfig.NamedStores.AlibabaCloud, 1)
expected := defaults.StorageConfig.AlibabaStorageConfig
expected.Bucket = bucketName
expected.Endpoint = "oss.test"
assert.Equal(t, expected, (alibaba.OssConfig)(nsCfg.AlibabaCloud["store-8"]))
})
}

@ -12,6 +12,8 @@ import (
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/grafana/dskit/flagext"
"github.com/grafana/loki/pkg/storage/chunk/cache"
"github.com/grafana/loki/pkg/storage/chunk/client"
"github.com/grafana/loki/pkg/storage/chunk/client/alibaba"
@ -64,16 +66,103 @@ type StoreLimits interface {
CardinalityLimit(string) int
}
// Storage configs defined as Named stores don't get any defaults as they do not
// register flags. To get around this we implement Unmarshaler interface that
// assigns the defaults before calling unmarshal.
// We cannot implement Unmarshaler directly on aws.StorageConfig or other stores
// as it would end up overriding values set as part of ApplyDynamicConfig().
// Note: we unmarshal a second time after applying dynamic configs
//
// Implementing the Unmarshaler for Named*StorageConfig types is fine as
// we do not apply any dynamic config on them.
type NamedAWSStorageConfig aws.StorageConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedAWSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*aws.StorageConfig)(cfg))
return unmarshal((*aws.StorageConfig)(cfg))
}
func (cfg *NamedAWSStorageConfig) Validate() error {
return (*aws.StorageConfig)(cfg).Validate()
}
type NamedBlobStorageConfig azure.BlobStorageConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedBlobStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*azure.BlobStorageConfig)(cfg))
return unmarshal((*azure.BlobStorageConfig)(cfg))
}
func (cfg *NamedBlobStorageConfig) Validate() error {
return (*azure.BlobStorageConfig)(cfg).Validate()
}
type NamedBOSStorageConfig baidubce.BOSStorageConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedBOSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*baidubce.BOSStorageConfig)(cfg))
return unmarshal((*baidubce.BOSStorageConfig)(cfg))
}
type NamedFSConfig local.FSConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedFSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*local.FSConfig)(cfg))
return unmarshal((*local.FSConfig)(cfg))
}
type NamedGCSConfig gcp.GCSConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedGCSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*gcp.GCSConfig)(cfg))
return unmarshal((*gcp.GCSConfig)(cfg))
}
type NamedOssConfig alibaba.OssConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedOssConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*alibaba.OssConfig)(cfg))
return unmarshal((*alibaba.OssConfig)(cfg))
}
type NamedSwiftConfig openstack.SwiftConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedSwiftConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*openstack.SwiftConfig)(cfg))
return unmarshal((*openstack.SwiftConfig)(cfg))
}
func (cfg *NamedSwiftConfig) Validate() error {
return (*openstack.SwiftConfig)(cfg).Validate()
}
type NamedCOSConfig ibmcloud.COSConfig
// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (cfg *NamedCOSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
flagext.DefaultValues((*ibmcloud.COSConfig)(cfg))
return unmarshal((*ibmcloud.COSConfig)(cfg))
}
// NamedStores helps configure additional object stores from a given storage provider
type NamedStores struct {
AWS map[string]aws.StorageConfig `yaml:"aws"`
Azure map[string]azure.BlobStorageConfig `yaml:"azure"`
BOS map[string]baidubce.BOSStorageConfig `yaml:"bos"`
Filesystem map[string]local.FSConfig `yaml:"filesystem"`
GCS map[string]gcp.GCSConfig `yaml:"gcs"`
AlibabaCloud map[string]alibaba.OssConfig `yaml:"alibabacloud"`
Swift map[string]openstack.SwiftConfig `yaml:"swift"`
COS map[string]ibmcloud.COSConfig `yaml:"cos"`
AWS map[string]NamedAWSStorageConfig `yaml:"aws"`
Azure map[string]NamedBlobStorageConfig `yaml:"azure"`
BOS map[string]NamedBOSStorageConfig `yaml:"bos"`
Filesystem map[string]NamedFSConfig `yaml:"filesystem"`
GCS map[string]NamedGCSConfig `yaml:"gcs"`
AlibabaCloud map[string]NamedOssConfig `yaml:"alibabacloud"`
Swift map[string]NamedSwiftConfig `yaml:"swift"`
COS map[string]NamedCOSConfig `yaml:"cos"`
// contains mapping from named store reference name to store type
storeType map[string]string `yaml:"-"`
@ -508,43 +597,47 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie
case config.StorageTypeAlibabaCloud:
ossCfg := cfg.AlibabaStorageConfig
if namedStore != "" {
var ok bool
ossCfg, ok = cfg.NamedStores.AlibabaCloud[namedStore]
nsCfg, ok := cfg.NamedStores.AlibabaCloud[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named alibabacloud oss storage config %s", name)
}
ossCfg = (alibaba.OssConfig)(nsCfg)
}
return alibaba.NewOssObjectClient(context.Background(), ossCfg)
case config.StorageTypeGCS:
gcsCfg := cfg.GCSConfig
if namedStore != "" {
var ok bool
gcsCfg, ok = cfg.NamedStores.GCS[namedStore]
nsCfg, ok := cfg.NamedStores.GCS[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named gcs storage config %s", name)
}
gcsCfg = (gcp.GCSConfig)(nsCfg)
}
return gcp.NewGCSObjectClient(context.Background(), gcsCfg, cfg.Hedging)
case config.StorageTypeAzure:
azureCfg := cfg.AzureStorageConfig
if namedStore != "" {
var ok bool
azureCfg, ok = cfg.NamedStores.Azure[namedStore]
nsCfg, ok := cfg.NamedStores.Azure[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named azure storage config %s", name)
}
azureCfg = (azure.BlobStorageConfig)(nsCfg)
}
return azure.NewBlobStorage(&azureCfg, clientMetrics.AzureMetrics, cfg.Hedging)
case config.StorageTypeSwift:
swiftCfg := cfg.Swift
if namedStore != "" {
var ok bool
swiftCfg, ok = cfg.NamedStores.Swift[namedStore]
nsCfg, ok := cfg.NamedStores.Swift[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named swift storage config %s", name)
}
swiftCfg = (openstack.SwiftConfig)(nsCfg)
}
return openstack.NewSwiftObjectClient(swiftCfg, cfg.Hedging)
@ -553,22 +646,24 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie
case config.StorageTypeFileSystem:
fsCfg := cfg.FSConfig
if namedStore != "" {
var ok bool
fsCfg, ok = cfg.NamedStores.Filesystem[namedStore]
nsCfg, ok := cfg.NamedStores.Filesystem[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named filesystem storage config %s", name)
}
fsCfg = (local.FSConfig)(nsCfg)
}
return local.NewFSObjectClient(fsCfg)
case config.StorageTypeBOS:
bosCfg := cfg.BOSStorageConfig
if namedStore != "" {
var ok bool
bosCfg, ok = cfg.NamedStores.BOS[namedStore]
nsCfg, ok := cfg.NamedStores.BOS[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named bos storage config %s", name)
}
bosCfg = (baidubce.BOSStorageConfig)(nsCfg)
}
return baidubce.NewBOSObjectStorage(&bosCfg)
@ -576,11 +671,12 @@ func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (clie
case config.StorageTypeCOS:
cosCfg := cfg.COSConfig
if namedStore != "" {
var ok bool
cosCfg, ok = cfg.NamedStores.COS[namedStore]
nsCfg, ok := cfg.NamedStores.COS[namedStore]
if !ok {
return nil, fmt.Errorf("Unrecognized named cos storage config %s", name)
}
cosCfg = (ibmcloud.COSConfig)(nsCfg)
}
return ibmcloud.NewCOSObjectClient(cosCfg, cfg.Hedging)
default:

@ -12,9 +12,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/loki/pkg/storage/chunk/client/aws"
"github.com/grafana/loki/pkg/storage/chunk/client/cassandra"
"github.com/grafana/loki/pkg/storage/chunk/client/gcp"
"github.com/grafana/loki/pkg/storage/chunk/client/local"
"github.com/grafana/loki/pkg/storage/config"
"github.com/grafana/loki/pkg/storage/stores/indexshipper"
@ -105,7 +103,7 @@ func TestNamedStores(t *testing.T) {
cfg := Config{
NamedStores: NamedStores{
Filesystem: map[string]local.FSConfig{
Filesystem: map[string]NamedFSConfig{
"named-store": {Directory: path.Join(tempDir, "named-store")},
},
},
@ -171,11 +169,11 @@ func TestNamedStores(t *testing.T) {
func TestNamedStores_populateStoreType(t *testing.T) {
t.Run("found duplicates", func(t *testing.T) {
ns := NamedStores{
AWS: map[string]aws.StorageConfig{
AWS: map[string]NamedAWSStorageConfig{
"store-1": {},
"store-2": {},
},
GCS: map[string]gcp.GCSConfig{
GCS: map[string]NamedGCSConfig{
"store-1": {},
},
}
@ -187,7 +185,7 @@ func TestNamedStores_populateStoreType(t *testing.T) {
t.Run("illegal store name", func(t *testing.T) {
ns := NamedStores{
GCS: map[string]gcp.GCSConfig{
GCS: map[string]NamedGCSConfig{
"aws": {},
},
}
@ -199,11 +197,11 @@ func TestNamedStores_populateStoreType(t *testing.T) {
t.Run("lookup populated entries", func(t *testing.T) {
ns := NamedStores{
AWS: map[string]aws.StorageConfig{
AWS: map[string]NamedAWSStorageConfig{
"store-1": {},
"store-2": {},
},
GCS: map[string]gcp.GCSConfig{
GCS: map[string]NamedGCSConfig{
"store-3": {},
},
}

@ -1022,7 +1022,7 @@ func TestStore_MultiPeriod(t *testing.T) {
},
TSDBShipperConfig: shipperConfig,
NamedStores: NamedStores{
Filesystem: map[string]local.FSConfig{
Filesystem: map[string]NamedFSConfig{
"named-store": {Directory: path.Join(tempDir, "named-store")},
},
},

@ -97,9 +97,11 @@ func (e ConfigEntry) Description() string {
}
type RootBlock struct {
Name string
Desc string
StructType reflect.Type
Name string
Desc string
// multiple entries are useful if the root blocks share the same
// underlying type
StructType []reflect.Type
}
func Flags(cfg flagext.Registerer) map[uintptr]*flag.Flag {
@ -629,8 +631,10 @@ func getFieldDescription(cfg interface{}, field reflect.StructField, fallback st
func isRootBlock(t reflect.Type, rootBlocks []RootBlock) (string, string, bool) {
for _, rootBlock := range rootBlocks {
if t == rootBlock.StructType {
return rootBlock.Name, rootBlock.Desc, true
for _, structType := range rootBlock.StructType {
if t == structType {
return rootBlock.Name, rootBlock.Desc, true
}
}
}

@ -49,109 +49,109 @@ var (
RootBlocks = []RootBlock{
{
Name: "server",
StructType: reflect.TypeOf(server.Config{}),
StructType: []reflect.Type{reflect.TypeOf(server.Config{})},
Desc: "Configures the server of the launched module(s).",
},
{
Name: "distributor",
StructType: reflect.TypeOf(distributor.Config{}),
StructType: []reflect.Type{reflect.TypeOf(distributor.Config{})},
Desc: "Configures the distributor.",
},
{
Name: "querier",
StructType: reflect.TypeOf(querier.Config{}),
StructType: []reflect.Type{reflect.TypeOf(querier.Config{})},
Desc: "Configures the querier. Only appropriate when running all modules or just the querier.",
},
{
Name: "query_scheduler",
StructType: reflect.TypeOf(scheduler.Config{}),
StructType: []reflect.Type{reflect.TypeOf(scheduler.Config{})},
Desc: "The query_scheduler block configures the Loki query scheduler. When configured it separates the tenant query queues from the query-frontend.",
},
{
Name: "frontend",
StructType: reflect.TypeOf(frontend.Config{}),
StructType: []reflect.Type{reflect.TypeOf(frontend.Config{})},
Desc: "The frontend block configures the Loki query-frontend.",
},
{
Name: "query_range",
StructType: reflect.TypeOf(queryrange.Config{}),
StructType: []reflect.Type{reflect.TypeOf(queryrange.Config{})},
Desc: "The query_range block configures the query splitting and caching in the Loki query-frontend.",
},
{
Name: "ruler",
StructType: reflect.TypeOf(ruler.Config{}),
StructType: []reflect.Type{reflect.TypeOf(ruler.Config{})},
Desc: "The ruler block configures the Loki ruler.",
},
{
Name: "ingester_client",
StructType: reflect.TypeOf(ingester_client.Config{}),
StructType: []reflect.Type{reflect.TypeOf(ingester_client.Config{})},
Desc: "The ingester_client block configures how the distributor will connect to ingesters. Only appropriate when running all components, the distributor, or the querier.",
},
{
Name: "ingester",
StructType: reflect.TypeOf(ingester.Config{}),
StructType: []reflect.Type{reflect.TypeOf(ingester.Config{})},
Desc: "The ingester block configures the ingester and how the ingester will register itself to a key value store.",
},
{
Name: "index_gateway",
StructType: reflect.TypeOf(indexgateway.Config{}),
StructType: []reflect.Type{reflect.TypeOf(indexgateway.Config{})},
Desc: "The index_gateway block configures the Loki index gateway server, responsible for serving index queries without the need to constantly interact with the object store.",
},
{
Name: "storage_config",
StructType: reflect.TypeOf(storage.Config{}),
StructType: []reflect.Type{reflect.TypeOf(storage.Config{})},
Desc: "The storage_config block configures one of many possible stores for both the index and chunks. Which configuration to be picked should be defined in schema_config block.",
},
{
Name: "chunk_store_config",
StructType: reflect.TypeOf(storage_config.ChunkStoreConfig{}),
StructType: []reflect.Type{reflect.TypeOf(storage_config.ChunkStoreConfig{})},
Desc: "The chunk_store_config block configures how chunks will be cached and how long to wait before saving them to the backing store.",
},
{
Name: "schema_config",
StructType: reflect.TypeOf(storage_config.SchemaConfig{}),
StructType: []reflect.Type{reflect.TypeOf(storage_config.SchemaConfig{})},
Desc: "Configures the chunk index schema and where it is stored.",
},
{
Name: "compactor",
StructType: reflect.TypeOf(compactor.Config{}),
StructType: []reflect.Type{reflect.TypeOf(compactor.Config{})},
Desc: "The compactor block configures the compactor component, which compacts index shards for performance.",
},
{
Name: "limits_config",
StructType: reflect.TypeOf(validation.Limits{}),
StructType: []reflect.Type{reflect.TypeOf(validation.Limits{})},
Desc: "The limits_config block configures global and per-tenant limits in Loki.",
},
{
Name: "frontend_worker",
StructType: reflect.TypeOf(querier_worker.Config{}),
StructType: []reflect.Type{reflect.TypeOf(querier_worker.Config{})},
Desc: "The frontend_worker configures the worker - running within the Loki querier - picking up and executing queries enqueued by the query-frontend.",
},
{
Name: "table_manager",
StructType: reflect.TypeOf(index.TableManagerConfig{}),
StructType: []reflect.Type{reflect.TypeOf(index.TableManagerConfig{})},
Desc: "The table_manager block configures the table manager for retention.",
},
{
Name: "runtime_config",
StructType: reflect.TypeOf(runtimeconfig.Config{}),
StructType: []reflect.Type{reflect.TypeOf(runtimeconfig.Config{})},
Desc: "Configuration for 'runtime config' module, responsible for reloading runtime configuration file.",
},
{
Name: "tracing",
StructType: reflect.TypeOf(tracing.Config{}),
StructType: []reflect.Type{reflect.TypeOf(tracing.Config{})},
Desc: "Configuration for tracing.",
},
{
Name: "analytics",
StructType: reflect.TypeOf(analytics.Config{}),
StructType: []reflect.Type{reflect.TypeOf(analytics.Config{})},
Desc: "Configuration for analytics.",
},
{
Name: "common",
StructType: reflect.TypeOf(common.Config{}),
StructType: []reflect.Type{reflect.TypeOf(common.Config{})},
Desc: "Common configuration to be shared between multiple modules. If a more specific configuration is given in other sections, the related configuration within this section will be ignored.",
},
@ -159,17 +159,17 @@ var (
// StoreConfig dskit type: https://github.com/grafana/dskit/blob/main/kv/client.go#L44-L52
{
Name: "consul",
StructType: reflect.TypeOf(consul.Config{}),
StructType: []reflect.Type{reflect.TypeOf(consul.Config{})},
Desc: "Configuration for a Consul client. Only applies if the selected kvstore is consul.",
},
{
Name: "etcd",
StructType: reflect.TypeOf(etcd.Config{}),
StructType: []reflect.Type{reflect.TypeOf(etcd.Config{})},
Desc: "Configuration for an ETCD v3 client. Only applies if the selected kvstore is etcd.",
},
{
Name: "memberlist",
StructType: reflect.TypeOf(memberlist.KVConfig{}),
StructType: []reflect.Type{reflect.TypeOf(memberlist.KVConfig{})},
Desc: `Configuration for memberlist client. Only applies if the selected kvstore is memberlist.
When a memberlist config with atleast 1 join_members is defined, kvstore of type memberlist is automatically selected for all the components that require a ring unless otherwise specified in the component's configuration section.`,
@ -177,77 +177,80 @@ When a memberlist config with atleast 1 join_members is defined, kvstore of type
// GRPC client
{
Name: "grpc_client",
StructType: reflect.TypeOf(grpcclient.Config{}),
StructType: []reflect.Type{reflect.TypeOf(grpcclient.Config{})},
Desc: "The grpc_client block configures the gRPC client used to communicate between two Loki components.",
},
// TLS config
{
Name: "tls_config",
StructType: reflect.TypeOf(tls.ClientConfig{}),
StructType: []reflect.Type{reflect.TypeOf(tls.ClientConfig{})},
Desc: "The TLS configuration.",
},
// Cache config
{
Name: "cache_config",
StructType: reflect.TypeOf(cache.Config{}),
StructType: []reflect.Type{reflect.TypeOf(cache.Config{})},
Desc: "The cache block configures the cache backend.",
},
// Schema periodic config
{
Name: "period_config",
StructType: reflect.TypeOf(storage_config.PeriodConfig{}),
StructType: []reflect.Type{reflect.TypeOf(storage_config.PeriodConfig{})},
Desc: "The period_config block configures what index schemas should be used for from specific time periods.",
},
// Storage config
{
Name: "aws_storage_config",
StructType: reflect.TypeOf(aws.StorageConfig{}),
Name: "aws_storage_config",
// aws.StorageConfig is the underlying type for storage.NamedAWSStorageConfig
// having these as separate block entries would result in duplicate storage configs
// similar reasoning applies to other storage configs listed below
StructType: []reflect.Type{reflect.TypeOf(aws.StorageConfig{}), reflect.TypeOf(storage.NamedAWSStorageConfig{})},
Desc: "The aws_storage_config block configures the connection to dynamoDB and S3 object storage. Either one of them or both can be configured.",
},
{
Name: "azure_storage_config",
StructType: reflect.TypeOf(azure.BlobStorageConfig{}),
StructType: []reflect.Type{reflect.TypeOf(azure.BlobStorageConfig{}), reflect.TypeOf(storage.NamedBlobStorageConfig{})},
Desc: "The azure_storage_config block configures the connection to Azure object storage backend.",
},
{
Name: "alibabacloud_storage_config",
StructType: reflect.TypeOf(alibaba.OssConfig{}),
StructType: []reflect.Type{reflect.TypeOf(alibaba.OssConfig{}), reflect.TypeOf(storage.NamedOssConfig{})},
Desc: "The alibabacloud_storage_config block configures the connection to Alibaba Cloud Storage object storage backend.",
},
{
Name: "gcs_storage_config",
StructType: reflect.TypeOf(gcp.GCSConfig{}),
StructType: []reflect.Type{reflect.TypeOf(gcp.GCSConfig{}), reflect.TypeOf(storage.NamedGCSConfig{})},
Desc: "The gcs_storage_config block configures the connection to Google Cloud Storage object storage backend.",
},
{
Name: "s3_storage_config",
StructType: reflect.TypeOf(aws.S3Config{}),
StructType: []reflect.Type{reflect.TypeOf(aws.S3Config{})},
Desc: "The s3_storage_config block configures the connection to Amazon S3 object storage backend.",
},
{
Name: "bos_storage_config",
StructType: reflect.TypeOf(baidubce.BOSStorageConfig{}),
StructType: []reflect.Type{reflect.TypeOf(baidubce.BOSStorageConfig{}), reflect.TypeOf(storage.NamedBOSStorageConfig{})},
Desc: "The bos_storage_config block configures the connection to Baidu Object Storage (BOS) object storage backend.",
},
{
Name: "swift_storage_config",
StructType: reflect.TypeOf(openstack.SwiftConfig{}),
StructType: []reflect.Type{reflect.TypeOf(openstack.SwiftConfig{}), reflect.TypeOf(storage.NamedSwiftConfig{})},
Desc: "The swift_storage_config block configures the connection to OpenStack Object Storage (Swift) object storage backend.",
},
{
Name: "cos_storage_config",
StructType: reflect.TypeOf(ibmcloud.COSConfig{}),
StructType: []reflect.Type{reflect.TypeOf(ibmcloud.COSConfig{}), reflect.TypeOf(storage.NamedCOSConfig{})},
Desc: "The cos_storage_config block configures the connection to IBM Cloud Object Storage (COS) backend.",
},
{
Name: "local_storage_config",
StructType: reflect.TypeOf(local.FSConfig{}),
StructType: []reflect.Type{reflect.TypeOf(local.FSConfig{}), reflect.TypeOf(storage.NamedFSConfig{})},
Desc: "The local_storage_config block configures the usage of local file system as object storage backend.",
},
{
Name: "named_stores_config",
StructType: reflect.TypeOf(storage.NamedStores{}),
StructType: []reflect.Type{reflect.TypeOf(storage.NamedStores{})},
Desc: `Configures additional object stores for a given storage provider.
Supported stores: aws, azure, bos, filesystem, gcs, swift.
Example:

Loading…
Cancel
Save