apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

operator: Allow SSE settings in S3 object storage secret (#10717)

Browse Source 
Co-authored-by: Robert Jacob <xperimental@solidproject.de>
pull/10519/head^2
Periklis Tsirakidis 2 years ago committed by GitHub
parent 93cd29f261
commit 50f838df90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 736 additions and 2 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      operator/CHANGELOG.md
  2. 26
      operator/docs/lokistack/object_storage.md
  3. 36
      operator/internal/handlers/internal/storage/secrets.go
  4. 68
      operator/internal/handlers/internal/storage/secrets_test.go
  5. 580
      operator/internal/manifests/internal/config/build_test.go
  6. 13
      operator/internal/manifests/internal/config/loki-config.yaml
  7. 14
      operator/internal/manifests/storage/options.go

1
operator/CHANGELOG.md
Unescape View File

@ -1,5 +1,6 @@
## Main
- [10717](https://github.com/grafana/loki/pull/10717) **periklis**: Allow SSE settings in AWS S3 object storage secret
- [10715](https://github.com/grafana/loki/pull/10715) **periklis**: Allow endpoint_suffix in azure object storage secret
- [10562](https://github.com/grafana/loki/pull/10562) **periklis**: Add memberlist IPv6 support
- [10720](https://github.com/grafana/loki/pull/10720) **JoaoBraveCoding**: Change default replication factor of 1x.medium to 2

26
operator/docs/lokistack/object_storage.md
Unescape View File

@ -38,6 +38,32 @@ _Note_: Upon setting up LokiStack for any object storage provider, you should co
--from-literal=region="<AWS_REGION_YOUR_BUCKET_LIVES_IN>"
```
or with `SSE-KMS` encryption
```console
kubectl create secret generic lokistack-dev-s3 \
--from-literal=bucketnames="<BUCKET_NAME>" \
--from-literal=endpoint="<AWS_BUCKET_ENDPOINT>" \
--from-literal=access_key_id="<AWS_ACCESS_KEY_ID>" \
--from-literal=access_key_secret="<AWS_ACCESS_KEY_SECRET>" \
--from-literal=sse_type="SSE-KMS" \
--from-literal=sse_kms_key_id="<AWS_SSE_KMS_KEY_ID>" \
--from-literal=sse_kms_encryption_context="<OPTIONAL_AWS_SSE_KMS_ENCRYPTION_CONTEXT_JSON>"
```
See also official docs on [AWS KMS Key ID](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#key-id) and [AWS KMS Encryption Context](https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#encrypt_context).
or with `SSE-S3` encryption
```console
kubectl create secret generic lokistack-dev-s3 \
--from-literal=bucketnames="<BUCKET_NAME>" \
--from-literal=endpoint="<AWS_BUCKET_ENDPOINT>" \
--from-literal=access_key_id="<AWS_ACCESS_KEY_ID>" \
--from-literal=access_key_secret="<AWS_ACCESS_KEY_SECRET>" \
--from-literal=sse_type="SSE-S3"
```
where `lokistack-dev-s3` is the secret name.
* Create an instance of [LokiStack](../hack/lokistack_dev.yaml) by referencing the secret name and type as `s3`:

36
operator/internal/handlers/internal/storage/secrets.go
Unescape View File

@ -96,7 +96,6 @@ func extractS3ConfigSecret(s *corev1.Secret) (*storage.S3StorageConfig, error) {
if len(buckets) == 0 {
return nil, kverrors.New("missing secret field", "field", "bucketnames")
}
// TODO buckets are comma-separated list
id := s.Data["access_key_id"]
if len(id) == 0 {
return nil, kverrors.New("missing secret field", "field", "access_key_id")
@ -109,12 +108,47 @@ func extractS3ConfigSecret(s *corev1.Secret) (*storage.S3StorageConfig, error) {
// Extract and validate optional fields
region := s.Data["region"]
sseCfg, err := extractS3SSEConfig(s.Data)
if err != nil {
return nil, err
}
return &storage.S3StorageConfig{
Endpoint: string(endpoint),
Buckets: string(buckets),
AccessKeyID: string(id),
AccessKeySecret: string(secret),
Region: string(region),
SSE: sseCfg,
}, nil
}
func extractS3SSEConfig(d map[string][]byte) (storage.S3SSEConfig, error) {
var (
sseType storage.S3SSEType
kmsKeyId, kmsEncryptionCtx string
)
switch sseType = storage.S3SSEType(d["sse_type"]); sseType {
case storage.SSEKMSType:
kmsEncryptionCtx = string(d["sse_kms_encryption_context"])
kmsKeyId = string(d["sse_kms_key_id"])
if kmsKeyId == "" {
return storage.S3SSEConfig{}, kverrors.New("missing secret field", "field", "sse_kms_key_id")
}
case storage.SSES3Type:
case "":
return storage.S3SSEConfig{}, nil
default:
return storage.S3SSEConfig{}, kverrors.New("unsupported secret field value (Supported: SSE-KMS, SSE-S3)", "field", "sse_type", "value", sseType)
}
return storage.S3SSEConfig{
Type: sseType,
KMSKeyID: string(kmsKeyId),
KMSEncryptionContext: string(kmsEncryptionCtx),
}, nil
}

68
operator/internal/handlers/internal/storage/secrets_test.go
Unescape View File

@ -181,7 +181,73 @@ func TestS3Extract(t *testing.T) {
wantErr: true,
},
{
name: "all set",
name: "unsupported SSE type",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
"access_key_secret": []byte("secret"),
"sse_type": []byte("unsupported"),
},
},
wantErr: true,
},
{
name: "missing SSE-KMS kms_key_id",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
"access_key_secret": []byte("secret"),
"sse_type": []byte("SSE-KMS"),
"sse_kms_encryption_context": []byte("kms-encryption-ctx"),
},
},
wantErr: true,
},
{
name: "all set with SSE-KMS",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
"access_key_secret": []byte("secret"),
"sse_type": []byte("SSE-KMS"),
"sse_kms_key_id": []byte("kms-key-id"),
},
},
},
{
name: "all set with SSE-KMS with encryption context",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
"access_key_secret": []byte("secret"),
"sse_type": []byte("SSE-KMS"),
"sse_kms_key_id": []byte("kms-key-id"),
"sse_kms_encryption_context": []byte("kms-encryption-ctx"),
},
},
},
{
name: "all set with SSE-S3",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),
"bucketnames": []byte("this,that"),
"access_key_id": []byte("id"),
"access_key_secret": []byte("secret"),
"sse_type": []byte("SSE-S3"),
},
},
},
{
name: "all set without SSE",
secret: &corev1.Secret{
Data: map[string][]byte{
"endpoint": []byte("here"),

580
operator/internal/manifests/internal/config/build_test.go
Unescape View File

@ -4057,3 +4057,583 @@ overrides:
require.YAMLEq(t, expCfg, string(cfg))
require.YAMLEq(t, expRCfg, string(rCfg))
}
func TestBuild_ConfigAndRuntimeConfig_WithS3SSEKMS(t *testing.T) {
expCfg := `
---
auth_enabled: true
chunk_store_config:
chunk_cache_config:
embedded_cache:
enabled: true
max_size_mb: 500
common:
storage:
s3:
s3: http://test.default.svc.cluster.local.:9000
bucketnames: loki
region: us-east
access_key_id: test
secret_access_key: test123
sse:
type: SSE-KMS
kms_key_id: test
kms_encryption_context: |
{"key": "value", "another":"value1"}
s3forcepathstyle: true
compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095
ring:
kvstore:
store: memberlist
heartbeat_period: 5s
heartbeat_timeout: 1m
instance_port: 9095
compactor:
compaction_interval: 2h
working_directory: /tmp/loki/compactor
frontend:
tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100
compress_responses: true
max_outstanding_per_tenant: 256
log_queries_longer_than: 5s
frontend_worker:
frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095
grpc_client_config:
max_send_msg_size: 104857600
match_max_concurrent: true
ingester:
chunk_block_size: 262144
chunk_encoding: snappy
chunk_idle_period: 1h
chunk_retain_period: 5m
chunk_target_size: 2097152
flush_op_timeout: 10m
lifecycler:
final_sleep: 0s
join_after: 30s
num_tokens: 512
ring:
replication_factor: 1
max_chunk_age: 2h
max_transfer_retries: 0
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
remote_timeout: 1s
# NOTE: Keep the order of keys as in Loki docs
# to enable easy diffs when vendoring newer
# Loki releases.
# (See https://grafana.com/docs/loki/latest/configuration/#limits_config)
#
# Values for not exposed fields are taken from the grafana/loki production
# configuration manifests.
# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet)
limits_config:
ingestion_rate_strategy: global
ingestion_rate_mb: 4
ingestion_burst_size_mb: 6
max_label_name_length: 1024
max_label_value_length: 2048
max_label_names_per_series: 30
reject_old_samples: true
reject_old_samples_max_age: 168h
creation_grace_period: 10m
enforce_metric_name: false
# Keep max_streams_per_user always to 0 to default
# using max_global_streams_per_user always.
# (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73)
max_streams_per_user: 0
max_line_size: 256000
max_entries_limit_per_query: 5000
max_global_streams_per_user: 0
max_chunks_per_query: 2000000
max_query_length: 721h
max_query_parallelism: 32
max_query_series: 500
cardinality_limit: 100000
max_streams_matchers_per_query: 1000
max_cache_freshness_per_query: 10m
per_stream_rate_limit: 3MB
per_stream_rate_limit_burst: 15MB
split_queries_by_interval: 30m
query_timeout: 1m
memberlist:
abort_if_cluster_join_fails: true
advertise_port: 7946
bind_port: 7946
join_members:
- loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946
max_join_backoff: 1m
max_join_retries: 10
min_join_backoff: 1s
querier:
engine:
max_look_back_period: 30s
extra_query_delay: 0s
max_concurrent: 2
query_ingesters_within: 3h
tail_max_duration: 1h
query_range:
align_queries_with_step: true
cache_results: true
max_retries: 5
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 500
parallelise_shardable_queries: true
schema_config:
configs:
- from: "2020-10-01"
index:
period: 24h
prefix: index_
object_store: s3
schema: v11
store: boltdb-shipper
server:
graceful_shutdown_timeout: 5s
grpc_server_min_time_between_pings: '10s'
grpc_server_ping_without_stream_allowed: true
grpc_server_max_concurrent_streams: 1000
grpc_server_max_recv_msg_size: 104857600
grpc_server_max_send_msg_size: 104857600
http_listen_port: 3100
http_server_idle_timeout: 30s
http_server_read_timeout: 30s
http_server_write_timeout: 10m0s
log_level: info
storage_config:
boltdb_shipper:
active_index_directory: /tmp/loki/index
cache_location: /tmp/loki/index_cache
cache_ttl: 24h
resync_interval: 5m
shared_store: s3
index_gateway_client:
server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095
tracing:
enabled: false
analytics:
reporting_enabled: false
`
expRCfg := `
---
overrides:
test-a:
ingestion_rate_mb: 2
ingestion_burst_size_mb: 5
max_global_streams_per_user: 1
max_chunks_per_query: 1000000
`
opts := Options{
Stack: lokiv1.LokiStackSpec{
Replication: &lokiv1.ReplicationSpec{
Factor: 1,
},
Limits: &lokiv1.LimitsSpec{
Global: &lokiv1.LimitsTemplateSpec{
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 4,
IngestionBurstSize: 6,
MaxLabelNameLength: 1024,
MaxLabelValueLength: 2048,
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
PerStreamRateLimit: 3,
PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
MaxChunksPerQuery: 2000000,
MaxQuerySeries: 500,
QueryTimeout: "1m",
CardinalityLimit: 100000,
},
},
Tenants: map[string]lokiv1.LimitsTemplateSpec{
"test-a": {
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 2,
IngestionBurstSize: 5,
MaxGlobalStreamsPerTenant: 1,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxChunksPerQuery: 1000000,
},
},
},
},
},
Overrides: map[string]LokiOverrides{
"test-a": {
Limits: lokiv1.LimitsTemplateSpec{
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 2,
MaxGlobalStreamsPerTenant: 1,
IngestionBurstSize: 5,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxChunksPerQuery: 1000000,
},
},
},
},
Namespace: "test-ns",
Name: "test",
Compactor: Address{
FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
FrontendWorker: Address{
FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
GossipRing: GossipRing{
InstancePort: 9095,
BindPort: 7946,
MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local",
},
Querier: Address{
Protocol: "http",
FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
Port: 3100,
},
IndexGateway: Address{
FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
StorageDirectory: "/tmp/loki",
MaxConcurrent: MaxConcurrent{
AvailableQuerierCPUCores: 2,
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
S3: &storage.S3StorageConfig{
Endpoint: "http://test.default.svc.cluster.local.:9000",
Region: "us-east",
Buckets: "loki",
AccessKeyID: "test",
AccessKeySecret: "test123",
SSE: storage.S3SSEConfig{
Type: storage.SSEKMSType,
KMSKeyID: "test",
KMSEncryptionContext: `{"key": "value", "another":"value1"}`,
},
},
Schemas: []lokiv1.ObjectStorageSchema{
{
Version: lokiv1.ObjectStorageSchemaV11,
EffectiveDate: "2020-10-01",
},
},
},
HTTPTimeouts: HTTPTimeoutConfig{
IdleTimeout: 30 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 10 * time.Minute,
},
}
cfg, rCfg, err := Build(opts)
require.NoError(t, err)
require.YAMLEq(t, expCfg, string(cfg))
require.YAMLEq(t, expRCfg, string(rCfg))
}
func TestBuild_ConfigAndRuntimeConfig_WithS3SSES3(t *testing.T) {
expCfg := `
---
auth_enabled: true
chunk_store_config:
chunk_cache_config:
embedded_cache:
enabled: true
max_size_mb: 500
common:
storage:
s3:
s3: http://test.default.svc.cluster.local.:9000
bucketnames: loki
region: us-east
access_key_id: test
secret_access_key: test123
sse:
type: SSE-S3
s3forcepathstyle: true
compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095
ring:
kvstore:
store: memberlist
heartbeat_period: 5s
heartbeat_timeout: 1m
instance_port: 9095
compactor:
compaction_interval: 2h
working_directory: /tmp/loki/compactor
frontend:
tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100
compress_responses: true
max_outstanding_per_tenant: 256
log_queries_longer_than: 5s
frontend_worker:
frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095
grpc_client_config:
max_send_msg_size: 104857600
match_max_concurrent: true
ingester:
chunk_block_size: 262144
chunk_encoding: snappy
chunk_idle_period: 1h
chunk_retain_period: 5m
chunk_target_size: 2097152
flush_op_timeout: 10m
lifecycler:
final_sleep: 0s
join_after: 30s
num_tokens: 512
ring:
replication_factor: 1
max_chunk_age: 2h
max_transfer_retries: 0
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
remote_timeout: 1s
# NOTE: Keep the order of keys as in Loki docs
# to enable easy diffs when vendoring newer
# Loki releases.
# (See https://grafana.com/docs/loki/latest/configuration/#limits_config)
#
# Values for not exposed fields are taken from the grafana/loki production
# configuration manifests.
# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet)
limits_config:
ingestion_rate_strategy: global
ingestion_rate_mb: 4
ingestion_burst_size_mb: 6
max_label_name_length: 1024
max_label_value_length: 2048
max_label_names_per_series: 30
reject_old_samples: true
reject_old_samples_max_age: 168h
creation_grace_period: 10m
enforce_metric_name: false
# Keep max_streams_per_user always to 0 to default
# using max_global_streams_per_user always.
# (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73)
max_streams_per_user: 0
max_line_size: 256000
max_entries_limit_per_query: 5000
max_global_streams_per_user: 0
max_chunks_per_query: 2000000
max_query_length: 721h
max_query_parallelism: 32
max_query_series: 500
cardinality_limit: 100000
max_streams_matchers_per_query: 1000
max_cache_freshness_per_query: 10m
per_stream_rate_limit: 3MB
per_stream_rate_limit_burst: 15MB
split_queries_by_interval: 30m
query_timeout: 1m
memberlist:
abort_if_cluster_join_fails: true
advertise_port: 7946
bind_port: 7946
join_members:
- loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946
max_join_backoff: 1m
max_join_retries: 10
min_join_backoff: 1s
querier:
engine:
max_look_back_period: 30s
extra_query_delay: 0s
max_concurrent: 2
query_ingesters_within: 3h
tail_max_duration: 1h
query_range:
align_queries_with_step: true
cache_results: true
max_retries: 5
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 500
parallelise_shardable_queries: true
schema_config:
configs:
- from: "2020-10-01"
index:
period: 24h
prefix: index_
object_store: s3
schema: v11
store: boltdb-shipper
server:
graceful_shutdown_timeout: 5s
grpc_server_min_time_between_pings: '10s'
grpc_server_ping_without_stream_allowed: true
grpc_server_max_concurrent_streams: 1000
grpc_server_max_recv_msg_size: 104857600
grpc_server_max_send_msg_size: 104857600
http_listen_port: 3100
http_server_idle_timeout: 30s
http_server_read_timeout: 30s
http_server_write_timeout: 10m0s
log_level: info
storage_config:
boltdb_shipper:
active_index_directory: /tmp/loki/index
cache_location: /tmp/loki/index_cache
cache_ttl: 24h
resync_interval: 5m
shared_store: s3
index_gateway_client:
server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095
tracing:
enabled: false
analytics:
reporting_enabled: false
`
expRCfg := `
---
overrides:
test-a:
ingestion_rate_mb: 2
ingestion_burst_size_mb: 5
max_global_streams_per_user: 1
max_chunks_per_query: 1000000
`
opts := Options{
Stack: lokiv1.LokiStackSpec{
Replication: &lokiv1.ReplicationSpec{
Factor: 1,
},
Limits: &lokiv1.LimitsSpec{
Global: &lokiv1.LimitsTemplateSpec{
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 4,
IngestionBurstSize: 6,
MaxLabelNameLength: 1024,
MaxLabelValueLength: 2048,
MaxLabelNamesPerSeries: 30,
MaxGlobalStreamsPerTenant: 0,
MaxLineSize: 256000,
PerStreamRateLimit: 3,
PerStreamRateLimitBurst: 15,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxEntriesLimitPerQuery: 5000,
MaxChunksPerQuery: 2000000,
MaxQuerySeries: 500,
QueryTimeout: "1m",
CardinalityLimit: 100000,
},
},
Tenants: map[string]lokiv1.LimitsTemplateSpec{
"test-a": {
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 2,
IngestionBurstSize: 5,
MaxGlobalStreamsPerTenant: 1,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxChunksPerQuery: 1000000,
},
},
},
},
},
Overrides: map[string]LokiOverrides{
"test-a": {
Limits: lokiv1.LimitsTemplateSpec{
IngestionLimits: &lokiv1.IngestionLimitSpec{
IngestionRate: 2,
MaxGlobalStreamsPerTenant: 1,
IngestionBurstSize: 5,
},
QueryLimits: &lokiv1.QueryLimitSpec{
MaxChunksPerQuery: 1000000,
},
},
},
},
Namespace: "test-ns",
Name: "test",
Compactor: Address{
FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
FrontendWorker: Address{
FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
GossipRing: GossipRing{
InstancePort: 9095,
BindPort: 7946,
MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local",
},
Querier: Address{
Protocol: "http",
FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
Port: 3100,
},
IndexGateway: Address{
FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
Port: 9095,
},
StorageDirectory: "/tmp/loki",
MaxConcurrent: MaxConcurrent{
AvailableQuerierCPUCores: 2,
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
S3: &storage.S3StorageConfig{
Endpoint: "http://test.default.svc.cluster.local.:9000",
Region: "us-east",
Buckets: "loki",
AccessKeyID: "test",
AccessKeySecret: "test123",
SSE: storage.S3SSEConfig{
Type: storage.SSES3Type,
KMSKeyID: "test",
KMSEncryptionContext: `{"key": "value", "another":"value1"}`,
},
},
Schemas: []lokiv1.ObjectStorageSchema{
{
Version: lokiv1.ObjectStorageSchemaV11,
EffectiveDate: "2020-10-01",
},
},
},
HTTPTimeouts: HTTPTimeoutConfig{
IdleTimeout: 30 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 10 * time.Minute,
},
}
cfg, rCfg, err := Build(opts)
require.NoError(t, err)
require.YAMLEq(t, expCfg, string(cfg))
require.YAMLEq(t, expRCfg, string(rCfg))
}

13
operator/internal/manifests/internal/config/loki-config.yaml
Unescape View File

@ -29,6 +29,19 @@ common:
region: {{ .Region }}
access_key_id: {{ .AccessKeyID }}
secret_access_key: {{ .AccessKeySecret }}
{{- with .SSE }}
{{- if .Type }}
sse:
type: {{ .Type }}
{{- if eq .Type "SSE-KMS" }}
kms_key_id: {{ .KMSKeyID }}
{{- with .KMSEncryptionContext }}
kms_encryption_context: |
{{ . }}
{{- end }}
{{- end}}
{{- end }}
{{- end }}
s3forcepathstyle: true
{{- end }}
{{- with .ObjectStorage.Swift }}

14
operator/internal/manifests/storage/options.go
Unescape View File

@ -41,6 +41,20 @@ type S3StorageConfig struct {
Buckets string
AccessKeyID string
AccessKeySecret string
SSE S3SSEConfig
}
type S3SSEType string
const (
SSEKMSType S3SSEType = "SSE-KMS"
SSES3Type S3SSEType = "SSE-S3"
)
type S3SSEConfig struct {
Type S3SSEType
KMSKeyID string
KMSEncryptionContext string
}
// SwiftStorageConfig for Swift storage config

Write Preview
Loading…
Cancel
Save