operator: Fix multiple storage support for boltdb_shipper and schema (#5699)

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [5699](https://github.com/grafana/loki/pull/5699) **Red-GV**: Configure boltdb_shipper and schema to use Azure, GCS, and Swift storage
 - [5701](https://github.com/grafana/loki/pull/5701) **sasagarw**: Make ReplicationFactor optional in LokiStack API
 - [5695](https://github.com/grafana/loki/pull/5695) **xperimental**: Update Go to 1.17
 - [5615](https://github.com/grafana/loki/pull/5615) **sasagarw**: Document how to connect to LokiStack gateway component

operator/internal/handlers/internal/secrets/secrets.go

@@ -36,17 +36,17 @@ func ExtractGatewaySecret(s *corev1.Secret, tenantName string) (*manifests.Tenan
 // ExtractStorageSecret reads a k8s secret into a manifest object storage struct if valid.
 func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorageSecretType) (*storage.Options, error) {
 	var err error
-	storage := storage.Options{}
+	storageOpts := storage.Options{SharedStore: secretType}
 
 	switch secretType {
 	case lokiv1beta1.ObjectStorageSecretAzure:
-		storage.Azure, err = extractAzureConfigSecret(s)
+		storageOpts.Azure, err = extractAzureConfigSecret(s)
 	case lokiv1beta1.ObjectStorageSecretGCS:
-		storage.GCS, err = extractGCSConfigSecret(s)
+		storageOpts.GCS, err = extractGCSConfigSecret(s)
 	case lokiv1beta1.ObjectStorageSecretS3:
-		storage.S3, err = extractS3ConfigSecret(s)
+		storageOpts.S3, err = extractS3ConfigSecret(s)
 	case lokiv1beta1.ObjectStorageSecretSwift:
-		storage.Swift, err = extractSwiftConfigSecret(s)
+		storageOpts.Swift, err = extractSwiftConfigSecret(s)
 	default:
 		return nil, kverrors.New("unknown secret type", "type", secretType)
 	}
@@ -54,7 +54,7 @@ func ExtractStorageSecret(s *corev1.Secret, secretType lokiv1beta1.ObjectStorage
 	if err != nil {
 		return nil, err
 	}
-	return &storage, nil
+	return &storageOpts, nil
 }
 
 func extractAzureConfigSecret(s *corev1.Secret) (*storage.AzureStorageConfig, error) {

operator/internal/manifests/internal/config/build_test.go

@@ -214,6 +214,7 @@ overrides:
 			IngesterMemoryRequest: 5000,
 		},
 		ObjectStorage: storage.Options{
+			SharedStore: lokiv1beta1.ObjectStorageSecretS3,
 			S3: &storage.S3StorageConfig{
 				Endpoint:        "http://test.default.svc.cluster.local.:9000",
 				Region:          "us-east",
@@ -452,6 +453,7 @@ overrides:
 			IngesterMemoryRequest: 5000,
 		},
 		ObjectStorage: storage.Options{
+			SharedStore: lokiv1beta1.ObjectStorageSecretS3,
 			S3: &storage.S3StorageConfig{
 				Endpoint:        "http://test.default.svc.cluster.local.:9000",
 				Region:          "us-east",
@@ -515,6 +517,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
 			IngesterMemoryRequest: 5000,
 		},
 		ObjectStorage: storage.Options{
+			SharedStore: lokiv1beta1.ObjectStorageSecretS3,
 			S3: &storage.S3StorageConfig{
 				Endpoint:        "http://test.default.svc.cluster.local.:9000",
 				Region:          "us-east",

operator/internal/manifests/internal/config/loki-config.yaml

@@ -152,7 +152,7 @@ schema_config:
       index:
         period: 24h
         prefix: index_
-      object_store: s3
+      object_store: {{ .ObjectStorage.SharedStore }}
       schema: v11
       store: boltdb-shipper
 server:
@@ -172,7 +172,7 @@ storage_config:
     cache_location: {{ .StorageDirectory }}/index_cache
     cache_ttl: 24h
     resync_interval: 5m
-    shared_store: s3
+    shared_store: {{ .ObjectStorage.SharedStore }}
     index_gateway_client:
       server_address: dns:///{{ .IndexGateway.FQDN }}:{{ .IndexGateway.Port }}
 tracing:

operator/internal/manifests/storage/options.go

@@ -1,12 +1,17 @@
 package storage
 
+import (
+	lokiv1beta1 "github.com/grafana/loki/operator/api/v1beta1"
+)
+
 // Options is used to configure Loki to integrate with
 // supported object storages.
 type Options struct {
-	Azure *AzureStorageConfig
-	GCS   *GCSStorageConfig
-	S3    *S3StorageConfig
-	Swift *SwiftStorageConfig
+	SharedStore lokiv1beta1.ObjectStorageSecretType
+	Azure       *AzureStorageConfig
+	GCS         *GCSStorageConfig
+	S3          *S3StorageConfig
+	Swift       *SwiftStorageConfig
 }
 
 // AzureStorageConfig for Azure storage config