From 7635a5cffa80cf5ff627b8de2ed00fa96c058629 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Fri, 8 Nov 2024 10:11:16 +0100 Subject: [PATCH] feat(operator): Add support for managed GCP WorkloadIdentity (#14752) --- .../loki-operator.clusterserviceversion.yaml | 2 +- .../loki-operator.clusterserviceversion.yaml | 2 +- .../loki-operator.clusterserviceversion.yaml | 2 +- .../loki-operator.clusterserviceversion.yaml | 2 +- operator/internal/config/managed_auth.go | 31 +++++- .../handlers/internal/storage/secrets.go | 18 ++-- .../handlers/internal/storage/secrets_test.go | 43 +++++++- .../manifests/openshift/credentialsrequest.go | 9 ++ .../internal/manifests/storage/configure.go | 19 +++- .../manifests/storage/configure_test.go | 97 +++++++++++++++++++ operator/internal/manifests/storage/var.go | 6 +- 11 files changed, 213 insertions(+), 18 deletions(-) diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index 88e32263e6..2eae710050 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -159,7 +159,7 @@ metadata: features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" - features.operators.openshift.io/token-auth-gcp: "false" + features.operators.openshift.io/token-auth-gcp: "true" operators.operatorframework.io/builder: operator-sdk-unknown operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 repository: https://github.com/grafana/loki/tree/main/operator diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index bccd4ce369..31c41588a0 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -166,7 +166,7 @@ metadata: features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" - features.operators.openshift.io/token-auth-gcp: "false" + features.operators.openshift.io/token-auth-gcp: "true" olm.skipRange: '>=5.9.0-0 <6.1.0' operatorframework.io/cluster-monitoring: "true" operatorframework.io/suggested-namespace: openshift-operators-redhat diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index 545fdedc90..4da83e32c0 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -16,7 +16,7 @@ metadata: features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" - features.operators.openshift.io/token-auth-gcp: "false" + features.operators.openshift.io/token-auth-gcp: "true" repository: https://github.com/grafana/loki/tree/main/operator support: Grafana Loki SIG Operator labels: diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index 4e78fabef9..f7a44332d7 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -22,7 +22,7 @@ metadata: features.operators.openshift.io/tls-profiles: "true" features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "true" - features.operators.openshift.io/token-auth-gcp: "false" + features.operators.openshift.io/token-auth-gcp: "true" olm.skipRange: '>=5.9.0-0 <6.1.0' operatorframework.io/cluster-monitoring: "true" operatorframework.io/suggested-namespace: openshift-operators-redhat diff --git a/operator/internal/config/managed_auth.go b/operator/internal/config/managed_auth.go index 6e3dc52471..4d3a07cdb0 100644 --- a/operator/internal/config/managed_auth.go +++ b/operator/internal/config/managed_auth.go @@ -1,6 +1,9 @@ package config -import "os" +import ( + "fmt" + "os" +) type AWSEnvironment struct { RoleARN string @@ -13,9 +16,15 @@ type AzureEnvironment struct { Region string } +type GCPEnvironment struct { + Audience string + ServiceAccountEmail string +} + type TokenCCOAuthConfig struct { AWS *AWSEnvironment Azure *AzureEnvironment + GCP *GCPEnvironment } func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { @@ -28,6 +37,12 @@ func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { subscriptionID := os.Getenv("SUBSCRIPTIONID") region := os.Getenv("REGION") + // GCP + projectNumber := os.Getenv("PROJECT_NUMBER") + poolID := os.Getenv("POOL_ID") + providerID := os.Getenv("PROVIDER_ID") + serviceAccountEmail := os.Getenv("SERVICE_ACCOUNT_EMAIL") + switch { case roleARN != "": return &TokenCCOAuthConfig{ @@ -44,6 +59,20 @@ func discoverTokenCCOAuthConfig() *TokenCCOAuthConfig { Region: region, }, } + case projectNumber != "" && poolID != "" && providerID != "" && serviceAccountEmail != "": + audience := fmt.Sprintf( + "//iam.googleapis.com/projects/%s/locations/global/workloadIdentityPools/%s/providers/%s", + projectNumber, + poolID, + providerID, + ) + + return &TokenCCOAuthConfig{ + GCP: &GCPEnvironment{ + Audience: audience, + ServiceAccountEmail: serviceAccountEmail, + }, + } } return nil diff --git a/operator/internal/handlers/internal/storage/secrets.go b/operator/internal/handlers/internal/storage/secrets.go index 36065afb4c..999c972b73 100644 --- a/operator/internal/handlers/internal/storage/secrets.go +++ b/operator/internal/handlers/internal/storage/secrets.go @@ -33,8 +33,7 @@ var ( errSecretUnknownSSEType = errors.New("unsupported SSE type (supported: SSE-KMS, SSE-S3)") errSecretHashError = errors.New("error calculating hash for secret") - errSecretUnknownCredentialMode = errors.New("unknown credential mode") - errSecretUnsupportedCredentialMode = errors.New("combination of storage type and credential mode not supported") + errSecretUnknownCredentialMode = errors.New("unknown credential mode") errAzureManagedIdentityNoOverride = errors.New("when in managed mode, storage secret can not contain credentials") errAzureInvalidEnvironment = errors.New("azure environment invalid (valid values: AzureGlobal, AzureChinaCloud, AzureGermanCloud, AzureUSGovernment)") @@ -47,6 +46,7 @@ var ( errGCPParseCredentialsFile = errors.New("gcp storage secret cannot be parsed from JSON content") errGCPWrongCredentialSourceFile = errors.New("credential source in secret needs to point to token file") + errGCPInvalidCredentialsFile = errors.New("gcp credentials file contains invalid fields") azureValidEnvironments = map[string]bool{ "AzureGlobal": true, @@ -355,6 +355,15 @@ func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMo } switch credentialMode { + case lokiv1.CredentialModeTokenCCO: + if _, ok := s.Data[storage.KeyGCPServiceAccountKeyFilename]; ok { + return nil, fmt.Errorf("%w: %s", errGCPInvalidCredentialsFile, "key.json must not be set for CredentialModeTokenCCO") + } + + return &storage.GCSStorageConfig{ + Bucket: string(bucket), + WorkloadIdentity: true, + }, nil case lokiv1.CredentialModeStatic: return &storage.GCSStorageConfig{ Bucket: string(bucket), @@ -380,12 +389,9 @@ func extractGCSConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMo WorkloadIdentity: true, Audience: audience, }, nil - case lokiv1.CredentialModeTokenCCO: - return nil, fmt.Errorf("%w: type: %s credentialMode: %s", errSecretUnsupportedCredentialMode, lokiv1.ObjectStorageSecretGCS, credentialMode) default: + return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode) } - - return nil, fmt.Errorf("%w: %s", errSecretUnknownCredentialMode, credentialMode) } func extractS3ConfigSecret(s *corev1.Secret, credentialMode lokiv1.CredentialMode) (*storage.S3StorageConfig, error) { diff --git a/operator/internal/handlers/internal/storage/secrets_test.go b/operator/internal/handlers/internal/storage/secrets_test.go index 0688b099f3..466c9f3487 100644 --- a/operator/internal/handlers/internal/storage/secrets_test.go +++ b/operator/internal/handlers/internal/storage/secrets_test.go @@ -277,6 +277,8 @@ func TestGCSExtract(t *testing.T) { type test struct { name string secret *corev1.Secret + tokenAuth *corev1.Secret + featureGates configv1.FeatureGates wantError string wantCredentialMode lokiv1.CredentialMode } @@ -343,6 +345,45 @@ func TestGCSExtract(t *testing.T) { }, wantCredentialMode: lokiv1.CredentialModeToken, }, + { + name: "invalid for token CCO", + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + TokenCCOAuthEnv: true, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + "key.json": []byte("{\"type\": \"external_account\", \"audience\": \"\", \"service_account_id\": \"\"}"), + }, + }, + wantError: "gcp credentials file contains invalid fields: key.json must not be set for CredentialModeTokenCCO", + }, + { + name: "valid for token CCO", + featureGates: configv1.FeatureGates{ + OpenShift: configv1.OpenShiftFeatureGates{ + Enabled: true, + TokenCCOAuthEnv: true, + }, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Data: map[string][]byte{ + "bucketname": []byte("here"), + }, + }, + tokenAuth: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "token-auth-config"}, + Data: map[string][]byte{ + "service_account.json": []byte("{\"type\": \"external_account\", \"audience\": \"test\", \"service_account_id\": \"\"}"), + }, + }, + wantCredentialMode: lokiv1.CredentialModeTokenCCO, + }, } for _, tst := range table { t.Run(tst.name, func(t *testing.T) { @@ -352,7 +393,7 @@ func TestGCSExtract(t *testing.T) { Type: lokiv1.ObjectStorageSecretGCS, } - opts, err := extractSecrets(spec, tst.secret, nil, configv1.FeatureGates{}) + opts, err := extractSecrets(spec, tst.secret, tst.tokenAuth, tst.featureGates) if tst.wantError == "" { require.NoError(t, err) require.Equal(t, tst.wantCredentialMode, opts.CredentialMode) diff --git a/operator/internal/manifests/openshift/credentialsrequest.go b/operator/internal/manifests/openshift/credentialsrequest.go index 3f20299821..7ed2476b3e 100644 --- a/operator/internal/manifests/openshift/credentialsrequest.go +++ b/operator/internal/manifests/openshift/credentialsrequest.go @@ -98,6 +98,15 @@ func encodeProviderSpec(env *config.TokenCCOAuthConfig) (*runtime.RawExtension, AzureSubscriptionID: azure.SubscriptionID, AzureTenantID: azure.TenantID, } + case env.GCP != nil: + spec = &cloudcredentialv1.GCPProviderSpec{ + PredefinedRoles: []string{ + "roles/iam.workloadIdentityUser", + "roles/storage.objectAdmin", + }, + Audience: env.GCP.Audience, + ServiceAccountEmail: env.GCP.ServiceAccountEmail, + } } encodedSpec, err := cloudcredentialv1.Codec.EncodeProviderSpec(spec.DeepCopyObject()) diff --git a/operator/internal/manifests/storage/configure.go b/operator/internal/manifests/storage/configure.go index ce6fa78273..a8d0bf69f8 100644 --- a/operator/internal/manifests/storage/configure.go +++ b/operator/internal/manifests/storage/configure.go @@ -141,7 +141,9 @@ func ensureObjectStoreCredentials(p *corev1.PodSpec, opts Options) corev1.PodSpe volumes = append(volumes, saTokenVolume(opts)) container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount) - if opts.OpenShift.TokenCCOAuthEnabled() && opts.S3 != nil && opts.S3.STS { + isSTS := opts.S3 != nil && opts.S3.STS + isWIF := opts.GCS != nil && opts.GCS.WorkloadIdentity + if opts.OpenShift.TokenCCOAuthEnabled() && (isSTS || isWIF) { volumes = append(volumes, tokenCCOAuthConfigVolume(opts)) container.VolumeMounts = append(container.VolumeMounts, tokenCCOAuthConfigVolumeMount) } @@ -223,8 +225,14 @@ func tokenAuthCredentials(opts Options) []corev1.EnvVar { envVarFromValue(EnvAzureFederatedTokenFile, ServiceAccountTokenFilePath), } case lokiv1.ObjectStorageSecretGCS: - return []corev1.EnvVar{ - envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)), + if opts.OpenShift.TokenCCOAuthEnabled() { + return []corev1.EnvVar{ + envVarFromValue(EnvGoogleApplicationCredentials, path.Join(tokenAuthConfigDirectory, KeyGCPManagedServiceAccountKeyFilename)), + } + } else { + return []corev1.EnvVar{ + envVarFromValue(EnvGoogleApplicationCredentials, path.Join(secretDirectory, KeyGCPServiceAccountKeyFilename)), + } } default: return []corev1.EnvVar{} @@ -326,7 +334,10 @@ func saTokenVolume(opts Options) corev1.Volume { audience = opts.Azure.Audience } case lokiv1.ObjectStorageSecretGCS: - audience = opts.GCS.Audience + audience = gcpDefaultAudience + if opts.GCS.Audience != "" { + audience = opts.GCS.Audience + } } return corev1.Volume{ Name: saTokenVolumeName, diff --git a/operator/internal/manifests/storage/configure_test.go b/operator/internal/manifests/storage/configure_test.go index 3080f924c1..2c4ef5636d 100644 --- a/operator/internal/manifests/storage/configure_test.go +++ b/operator/internal/manifests/storage/configure_test.go @@ -689,6 +689,103 @@ func TestConfigureDeploymentForStorageType(t *testing.T) { }, }, }, + { + desc: "object storage GCS with Workload Identity and OpenShift Managed Credentials", + opts: Options{ + SecretName: "test", + SharedStore: lokiv1.ObjectStorageSecretGCS, + CredentialMode: lokiv1.CredentialModeTokenCCO, + GCS: &GCSStorageConfig{ + WorkloadIdentity: true, + }, + OpenShift: OpenShiftOptions{ + Enabled: true, + CloudCredentials: CloudCredentials{ + SecretName: "cloud-credentials", + SHA1: "deadbeef", + }, + }, + }, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "loki-ingester", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + ReadOnly: false, + MountPath: "/etc/storage/secrets", + }, + { + Name: saTokenVolumeName, + ReadOnly: false, + MountPath: saTokenVolumeMountPath, + }, + tokenCCOAuthConfigVolumeMount, + }, + Env: []corev1.EnvVar{ + { + Name: EnvGoogleApplicationCredentials, + Value: "/etc/storage/token-auth/service_account.json", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test", + }, + }, + }, + { + Name: saTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: gcpDefaultAudience, + ExpirationSeconds: ptr.To[int64](3600), + Path: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + }, + { + Name: tokenAuthConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "cloud-credentials", + }, + }, + }, + }, + }, + }, + }, + }, + }, { desc: "object storage S3", opts: Options{ diff --git a/operator/internal/manifests/storage/var.go b/operator/internal/manifests/storage/var.go index ccb69ff272..108d811412 100644 --- a/operator/internal/manifests/storage/var.go +++ b/operator/internal/manifests/storage/var.go @@ -97,6 +97,8 @@ const ( KeyGCPStorageBucketName = "bucketname" // KeyGCPServiceAccountKeyFilename is the service account key filename containing the Google authentication credentials. KeyGCPServiceAccountKeyFilename = "key.json" + // KeyGCPManagedServiceAccountKeyFilename is the service account key filename for the managed Google service account. + KeyGCPManagedServiceAccountKeyFilename = "service_account.json" // KeySwiftAuthURL is the secret data key for the OpenStack Swift authentication URL. KeySwiftAuthURL = "auth_url" @@ -140,9 +142,9 @@ const ( tokenAuthConfigVolumeName = "token-auth-config" tokenAuthConfigDirectory = "/etc/storage/token-auth" - awsDefaultAudience = "sts.amazonaws.com" - + awsDefaultAudience = "sts.amazonaws.com" azureDefaultAudience = "api://AzureADTokenExchange" + gcpDefaultAudience = "openshift" azureManagedCredentialKeyClientID = "azure_client_id" azureManagedCredentialKeyTenantID = "azure_tenant_id"